From df96044338ae6ea6266915e5fe6b1c68b83a0307 Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Mon, 14 Sep 2026 22:36:05 +0800 Subject: [PATCH 01/27] fix(extraction): stop simple-LLM entity extraction from leaking rules/enum values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two concrete failure modes in the 简易LLM ontology build path: business rules (e.g. "促销定价规则") got extracted as entities instead of logic_rules, and enumeration values of a concept's property (信用分层's "A层"/"B层", 逾期阶段's "M0"/"M2+", 融资轮次's "D轮融资") got extracted as standalone entities/instances instead of being recognized as values, not concepts. The ontology's entity layer is meant to hold unified concept-level types only (per graph-engineering discipline already partially applied here via CONCEPT_INSTANCE_DIRECTIVE), and neither existing directive's examples (named objects: company/person/order names) covered these two shapes. Two-layer fix, matching the existing prompt+deterministic-backstop pattern: - Prompt layer: explicit negative rules added to both CONCEPT_INSTANCE_DIRECTIVE and the shared extract_ontology user message, with concrete examples of both failure modes. - Backstop layer: normalize_extracted_ontology now drops entities matching _looks_like_rule_name (business-rule/policy suffixes) or _looks_like_enum_value (tier/stage/round-marker shape) — the same place junk entity names were already being dropped. Relations left dangling by a dropped entity are removed by the existing dangling-reference filter, no extra handling needed. Also extends the document-extraction eval harness (evals/document_extraction/validators.py) with the same enum-value pattern in instance_leakage_gate and a new rule_leakage_gate, so this regression class is caught automatically going forward — the existing _ID_PATTERN/_TRAILING_DIGITS_PATTERN gate only caught bare-digit IDs (CUS0001, 客户001), not letter+marker shapes like "A层" or "M2+". Updates the pre-existing test that had encoded the old (wrong) expected behavior: it previously asserted "A层"/"B层" should survive as entities with INSTANCE-OF relations, which is exactly the leakage this fix corrects. --- backend/app/services/llm_service.py | 31 +++++++++++++- backend/app/tasks/extraction.py | 10 ++++- backend/evals/document_extraction/run.py | 4 +- .../evals/document_extraction/validators.py | 34 ++++++++++++++- .../agent/test_extraction_normalization.py | 42 ++++++++++++++++--- .../test_document_extraction_validators.py | 42 ++++++++++++++++++- 6 files changed, 152 insertions(+), 11 deletions(-) diff --git a/backend/app/services/llm_service.py b/backend/app/services/llm_service.py index 7f9bfdf6..c4d1b904 100644 --- a/backend/app/services/llm_service.py +++ b/backend/app/services/llm_service.py @@ -31,7 +31,12 @@ def extract_ontology(text: str, prompt_content: str, model_config: dict, model_n "每条关系的 source/target 必须取自已提取的实体\n" "5. 逻辑规则按 Palantir Ontology Functions 模型分类为 derived_property/aggregation/" "complex_edit/external_query 之一,写入 function_type + definition 字段,禁止输出 " - "IF-THEN 自然语言公式\n\n" + "IF-THEN 自然语言公式\n" + "6. 业务规则、政策、判断条件、定价/审批/校验类规则(如\"促销定价规则\")不是实体," + "禁止作为 entity 或 instance 输出,必须放入 logic_rules 数组\n" + "7. 属性的枚举值/分级/分档/阶段标签(如信用分层的\"A层\"\"B层\"、逾期阶段的\"M0\"\"M1\"、" + "融资轮次的\"A轮融资\")不是独立概念也不是命名实例,不要为它们创建 entity 或 instance," + "只需体现为所属概念某个属性的枚举值\n\n" f"文档内容:\n\n{text}" )}, ] @@ -61,7 +66,7 @@ def normalize_extracted_ontology(parsed: Any) -> dict: if not isinstance(entity, dict): continue name = _clean_entity_name(entity.get("name_cn")) - if not name or _is_junk_entity_name(name): + if not name or _is_junk_entity_name(name) or _looks_like_rule_name(name) or _looks_like_enum_value(name): continue key = _normalize_name(name) canonical = seen_names.get(key) @@ -140,6 +145,28 @@ def _is_junk_entity_name(name: str) -> bool: return False +_RULE_NAME_SUFFIXES = ("规则", "政策", "办法", "制度", "准则", "细则", "规程") + + +def _looks_like_rule_name(name: str) -> bool: + """A business rule/policy (e.g. "促销定价规则") is not an entity — it + belongs in logic_rules, not the concept graph.""" + return name.endswith(_RULE_NAME_SUFFIXES) + + +# A letter (+ optional 1-2 digits) followed by a small closed set of +# tier/stage/round markers, and nothing else — the shape of an enumeration +# VALUE of some concept's property (信用分层's "A层", 逾期阶段's "M0"/"M1+", +# 融资轮次's "D轮融资"), not a concept in its own right. +_ENUM_VALUE_PATTERN = re.compile( + r"^[A-Za-z]{1,2}[+-]?(轮融资|阶段|层|级|档|类)$|^M\d{1,2}\+?(阶段)?$" +) + + +def _looks_like_enum_value(name: str) -> bool: + return bool(_ENUM_VALUE_PATTERN.match(name)) + + def resolve_entities(entities: list[dict], model_config: dict, model_name: str) -> dict[str, str]: """Cross-document entity resolution (graph-engineering playbook, resolution stage): within each entity type, cluster surface-form variants of the same diff --git a/backend/app/tasks/extraction.py b/backend/app/tasks/extraction.py index fa4c21ae..7fb5e44f 100644 --- a/backend/app/tasks/extraction.py +++ b/backend/app/tasks/extraction.py @@ -20,7 +20,15 @@ 不要为文档中提到的每个具体命名对象(如具体公司名、具体人名、具体订单号、具体条款标题)单独创建 entity。 文档中提到的具体命名实例,放入新增的 instances 数组,通过 entity_type 字段关联到其所属概念的 name_cn: "instances": [{"entity_type": "所属概念的name_cn", "name_cn": "实例名称", "name_en": "可选", "properties": {"属性1": "值"}, "confidence": 0.9}] -relations 只在概念之间建立(如 供应商 -[supply]-> 产品),不要在具体实例之间建立关系。""" +relations 只在概念之间建立(如 供应商 -[supply]-> 产品),不要在具体实例之间建立关系。 + +属性的枚举值/分级/分档/阶段标签(如信用分层的"A层""B层""C层"、逾期阶段的"M0""M1""M2+"、 +融资轮次的"A轮融资""D轮融资")不是独立的概念,也不是需要单独建 instance 的命名对象——它们是所属 +概念(如"信用分层""逾期阶段""融资")某个属性的可选取值。不要为它们创建 entity,也不要创建 +instance;只需在所属概念的 properties 里以枚举值形式体现(如信用分层.properties.levels = ["A层","B层","C层"])。 + +业务规则、政策、判断条件、定价/审批/校验类规则(如"促销定价规则""审批超时规则""授信额度校验") +不是实体,禁止作为 entity 或 instance 输出,必须放入 logic_rules 数组。""" # ── Extraction results merge (方案A) ─────────────────────────────────────── diff --git a/backend/evals/document_extraction/run.py b/backend/evals/document_extraction/run.py index 2a072e3a..4674df8d 100644 --- a/backend/evals/document_extraction/run.py +++ b/backend/evals/document_extraction/run.py @@ -4,7 +4,7 @@ import json import os -from evals.document_extraction.validators import dedup_gate, instance_leakage_gate, keyword_recall +from evals.document_extraction.validators import dedup_gate, instance_leakage_gate, keyword_recall, rule_leakage_gate def run_eval(manifest_path: str, ground_truth_dir: str, adapter, repo_root: str) -> dict: @@ -26,6 +26,7 @@ def run_eval(manifest_path: str, ground_truth_dir: str, adapter, repo_root: str) "keyword_recall": keyword_recall(ground_truth, extraction_result), "dedup_findings": dedup_gate(extraction_result), "leakage_findings": instance_leakage_gate(extraction_result), + "rule_leakage_findings": rule_leakage_gate(extraction_result), } except Exception as e: case_entry = { @@ -46,6 +47,7 @@ def run_eval(manifest_path: str, ground_truth_dir: str, adapter, repo_root: str) "mean_keyword_recall": mean_recall, "cases_with_dedup_findings": sum(1 for c in case_results if c.get("dedup_findings")), "cases_with_leakage_findings": sum(1 for c in case_results if c.get("leakage_findings")), + "cases_with_rule_leakage_findings": sum(1 for c in case_results if c.get("rule_leakage_findings")), } return {"cases": case_results, "summary": summary} diff --git a/backend/evals/document_extraction/validators.py b/backend/evals/document_extraction/validators.py index c3b311fe..91079689 100644 --- a/backend/evals/document_extraction/validators.py +++ b/backend/evals/document_extraction/validators.py @@ -13,6 +13,17 @@ _ID_PATTERN = re.compile(r"^[A-Za-z]+\d+$") _TRAILING_DIGITS_PATTERN = re.compile(r"\d{3,}$") +# A letter (+ optional 1-2 digits) followed by a small closed set of +# tier/stage/round markers, and nothing else — the shape of an enumeration +# VALUE of some concept's property (信用分层's "A层", 逾期阶段's "M2+", +# 融资轮次's "D轮融资"), not a concept in its own right. Kept in sync with +# app.services.llm_service._looks_like_enum_value. +_ENUM_VALUE_PATTERN = re.compile( + r"^[A-Za-z]{1,2}[+-]?(轮融资|阶段|层|级|档|类)$|^M\d{1,2}\+?(阶段)?$" +) + +_RULE_NAME_SUFFIXES = ("规则", "政策", "办法", "制度", "准则", "细则", "规程") + def _all_text_blobs(extraction_result: dict) -> list[str]: """Every name_cn/description/entity_type string across all categories, @@ -67,12 +78,33 @@ def _looks_like_instance_id(name: str) -> bool: return bool(_ID_PATTERN.match(name)) or bool(_TRAILING_DIGITS_PATTERN.search(name)) +def _looks_like_enum_value(name: str) -> bool: + return bool(_ENUM_VALUE_PATTERN.match(name)) + + def instance_leakage_gate(extraction_result: dict) -> list[dict]: findings: list[dict] = [] for item in extraction_result.get("entities") or []: if not isinstance(item, dict): continue name = str(item.get("name_cn") or "") - if name and _looks_like_instance_id(name): + if not name: + continue + if _looks_like_instance_id(name): findings.append({"name": name, "reason": "matches ID/numbered-record pattern"}) + elif _looks_like_enum_value(name): + findings.append({"name": name, "reason": "matches tier/stage/round enum-value pattern"}) + return findings + + +def rule_leakage_gate(extraction_result: dict) -> list[dict]: + """A business rule/policy name (e.g. "促销定价规则") is not an entity — + it belongs in logic_rules.""" + findings: list[dict] = [] + for item in extraction_result.get("entities") or []: + if not isinstance(item, dict): + continue + name = str(item.get("name_cn") or "") + if name and name.endswith(_RULE_NAME_SUFFIXES): + findings.append({"name": name, "reason": "matches business-rule/policy name pattern"}) return findings diff --git a/backend/tests/agent/test_extraction_normalization.py b/backend/tests/agent/test_extraction_normalization.py index 43ba1907..917c0a0e 100644 --- a/backend/tests/agent/test_extraction_normalization.py +++ b/backend/tests/agent/test_extraction_normalization.py @@ -15,6 +15,8 @@ from app.services.llm_service import ( VAGUE_RELATION_TYPES, + _looks_like_enum_value, + _looks_like_rule_name, apply_entity_resolution, extract_ontology, normalize_extracted_ontology, @@ -39,7 +41,10 @@ def test_extraction_discipline_red_contract(): def test_normalization_dedups_and_drops_junk_over_test_data_document(): """A noisy extraction over the real 信贷 strategy document is normalized: duplicate surface forms merge, junk entities vanish, dangling and vague - relations are dropped, hierarchy relations survive.""" + relations are dropped, hierarchy relations survive. Enum-value labels + (tier/stage/round markers) and business-rule names are also entity + leakage — dropped, along with any relation left dangling by their + removal.""" document = TEST_DATA / "信贷" / "信贷业务战略.md" if not document.exists(): pytest.skip("test_data document unavailable") @@ -54,6 +59,7 @@ def test_normalization_dedups_and_drops_junk_over_test_data_document(): {"name_cn": "A层", "type": "Category", "description": "优质客户", "properties": {}}, {"name_cn": "B层", "type": "Category", "description": "良好客户", "properties": {}}, {"name_cn": "授信额度", "type": "Concept", "description": "可贷上限", "properties": {}}, + {"name_cn": "促销定价规则", "type": "Concept", "description": "误提取的规则", "properties": {}}, {"name_cn": "12345", "type": "Concept", "description": "噪音", "properties": {}}, {"name_cn": "", "type": "Concept", "description": "空名", "properties": {}}, {"name_cn": "信贷业务战略.md", "type": "Concept", "description": "文件名", "properties": {}}, @@ -85,6 +91,11 @@ def test_normalization_dedups_and_drops_junk_over_test_data_document(): # junk entities dropped assert "12345" not in names assert "信贷业务战略.md" not in names + # enum-value labels (tier markers) are not concepts — dropped + assert "A层" not in names + assert "B层" not in names + # a business rule is not an entity — dropped (belongs in logic_rules) + assert "促销定价规则" not in names assert not any(not (e.get("name_cn") or "").strip() for e in normalized["entities"]) # every surviving entity carries a grounded one-line description assert all(e.get("description") for e in normalized["entities"]) @@ -97,10 +108,31 @@ def test_normalization_dedups_and_drops_junk_over_test_data_document(): assert ("借款人", "DEPENDS_ON", "信用评分") in triples # the duplicate DEPENDS_ON appeared twice -> deduplicated to one assert sum(1 for r in normalized["relations"] if r["type"] == "DEPENDS_ON") == 1 - # hierarchy relations (IS-A / INSTANCE-OF) survive intact - assert ("A层", "INSTANCE-OF", "信用分层") in triples - assert ("B层", "INSTANCE-OF", "信用分层") in triples - assert ("A层", "GOVERNED_BY", "授信额度") in triples + # relations pointing at the dropped enum-value entities are now dangling + # references and are dropped too + assert ("A层", "INSTANCE-OF", "信用分层") not in triples + assert ("B层", "INSTANCE-OF", "信用分层") not in triples + assert ("A层", "GOVERNED_BY", "授信额度") not in triples + + +def test_looks_like_enum_value_matches_tier_stage_and_round_labels(): + for name in ("A层", "B层", "C级", "M0", "M1", "M2+", "M0阶段", "A轮融资", "D轮融资", "A档", "B类"): + assert _looks_like_enum_value(name), name + + +def test_looks_like_enum_value_does_not_match_real_concepts(): + for name in ("借款人", "供应商", "A股", "P7", "信用分层", "授信额度", "D轮"): + assert not _looks_like_enum_value(name), name + + +def test_looks_like_rule_name_matches_policy_and_rule_suffixes(): + for name in ("促销定价规则", "审批超时规则", "员工考勤制度", "风险准入政策"): + assert _looks_like_rule_name(name), name + + +def test_looks_like_rule_name_does_not_match_concepts(): + for name in ("借款人", "信用分层", "贷款产品"): + assert not _looks_like_rule_name(name), name def test_normalization_is_idempotent_and_keeps_clean_output(): diff --git a/backend/tests/evals/test_document_extraction_validators.py b/backend/tests/evals/test_document_extraction_validators.py index 46be2409..acc813ee 100644 --- a/backend/tests/evals/test_document_extraction_validators.py +++ b/backend/tests/evals/test_document_extraction_validators.py @@ -3,7 +3,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from evals.document_extraction.validators import keyword_recall, dedup_gate, instance_leakage_gate +from evals.document_extraction.validators import keyword_recall, dedup_gate, instance_leakage_gate, rule_leakage_gate SAMPLE_RESULT = { "entities": [ @@ -101,3 +101,43 @@ def test_instance_leakage_gate_clean_for_concept_names(): "relations": [], "logic_rules": [], "actions": [], "instances": [], } assert instance_leakage_gate(result) == [] + + +def test_instance_leakage_gate_flags_enum_value_entity_names(): + result = { + "entities": [ + {"name_cn": "信用分层", "description": "概念实体"}, + {"name_cn": "A层", "description": "leaked enum value"}, + {"name_cn": "M2+", "description": "leaked enum value"}, + {"name_cn": "D轮融资", "description": "leaked enum value"}, + ], + "relations": [], "logic_rules": [], "actions": [], "instances": [], + } + findings = instance_leakage_gate(result) + flagged_names = {f["name"] for f in findings} + assert flagged_names == {"A层", "M2+", "D轮融资"} + + +def test_rule_leakage_gate_flags_business_rule_entity_names(): + result = { + "entities": [ + {"name_cn": "借款人", "description": "概念实体"}, + {"name_cn": "促销定价规则", "description": "leaked business rule"}, + {"name_cn": "风险准入政策", "description": "leaked business rule"}, + ], + "relations": [], "logic_rules": [], "actions": [], "instances": [], + } + findings = rule_leakage_gate(result) + flagged_names = {f["name"] for f in findings} + assert flagged_names == {"促销定价规则", "风险准入政策"} + + +def test_rule_leakage_gate_clean_for_concept_names(): + result = { + "entities": [ + {"name_cn": "借款人", "description": "概念实体"}, + {"name_cn": "信用分层", "description": "概念实体"}, + ], + "relations": [], "logic_rules": [], "actions": [], "instances": [], + } + assert rule_leakage_gate(result) == [] From 02408db65d4a4bb6ff21b784ecc8ec2d38f9e4e1 Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Tue, 15 Sep 2026 12:07:52 +0800 Subject: [PATCH 02/27] fix(extraction): drop dangling linked_entities references to rule-category names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit logic_rules/actions.linked_entities was saved straight from the raw LLM output, unfiltered. Since the previous commit correctly stopped extracting rule-category labels (e.g. "反欺诈规则") as entities, any rule/action whose linked_entities still named one of those labels now produced a dangling reference — confirmed on a real run: 101 P0 validation warnings on deepseek-v4-pro, 9 on deepseek-v4-flash, all LOGIC_BROKEN_ENTITY_REF/ACTION_BROKEN_ENTITY_REF pointing at names that are correctly absent from entities. Filters both fields down to names present in entity_name_to_id (the authoritative saved-entity set built earlier in the same function), mirroring the dangling-relation cleanup already applied to `relations` a few lines above. --- backend/app/tasks/extraction.py | 9 ++++ .../tests/agent/test_simple_llm_instances.py | 45 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/backend/app/tasks/extraction.py b/backend/app/tasks/extraction.py index 7fb5e44f..bc6bf7cb 100644 --- a/backend/app/tasks/extraction.py +++ b/backend/app/tasks/extraction.py @@ -592,6 +592,12 @@ def _match_logic_rules(text: str, logic_name_to_id: dict) -> list: if not llm_linked: combined = " ".join(filter(None, [name_cn, r_data.get("definition",""), r_data.get("description","")])) llm_linked = _match_entities(combined, all_entity_names) + # A rule-category label (e.g. "反欺诈规则") is not a concept entity — + # it belongs in logic_rules, not entities — so a linked_entities + # reference to one is a dangling reference, same as a relation + # pointing at an entity the model never extracted; drop it rather + # than let it surface as a broken-reference validation warning. + llm_linked = [n for n in llm_linked if n in entity_name_to_id] if name_cn in existing_rule_map: rule = existing_rule_map[name_cn] @@ -631,6 +637,9 @@ def _match_logic_rules(text: str, logic_name_to_id: dict) -> list: if not linked_ents: combined = " ".join(filter(None, [name_cn, a_data.get("description","")])) linked_ents = _match_entities(combined, all_entity_names) + # Same dangling-reference cleanup as logic_rules above — a + # rule-category label is not a concept entity. + linked_ents = [n for n in linked_ents if n in entity_name_to_id] linked_logic_names = a_data.get("linked_logic_names", []) linked_ids = [logic_name_to_id[n] for n in linked_logic_names if n in logic_name_to_id] diff --git a/backend/tests/agent/test_simple_llm_instances.py b/backend/tests/agent/test_simple_llm_instances.py index 7b1333f5..cc2545f9 100644 --- a/backend/tests/agent/test_simple_llm_instances.py +++ b/backend/tests/agent/test_simple_llm_instances.py @@ -4,6 +4,7 @@ instance data attached to its concept entities (row_identity + row_data), consistent with Pipeline/Mapping output — not just Entity/Relation rows. """ +import json import os import subprocess import sys @@ -237,3 +238,47 @@ def test_simple_llm_without_instances_still_succeeds(schema, monkeypatch): )).scalar_one() == 0 finally: session.close() + + +def test_dangling_linked_entities_reference_is_dropped(schema, monkeypatch): + """A logic_rule/action whose linked_entities names a rule-category label + (e.g. "反欺诈规则") rather than a real concept entity must not carry that + dangling reference into the saved row — the same discipline already + applied to relations pointing at an entity the model never extracted.""" + factory, session = _worker(schema) + task_id = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + monkeypatch.setattr( + "app.services.llm_service.extract_ontology", + lambda *a, **k: { + "entities": [{"name_cn": "设备指纹", "type": "Concept", "description": "设备指纹特征", "properties": {}}], + "relations": [], + "logic_rules": [ + {"name_cn": "设备指纹黑名单拒绝", "function_type": "derived_property", "definition": "命中黑名单则拒绝", + "linked_entities": ["设备指纹", "反欺诈规则"]}, + ], + "actions": [ + {"name_cn": "标记欺诈", "rules": [], "linked_entities": ["设备指纹", "反欺诈规则"]}, + ], + }, + ) + extraction_task.run_extraction(task_id) + try: + status = session.execute(text( + "SELECT status, error FROM extraction_tasks WHERE id = :id" + ), {"id": task_id}).mappings().one() + assert status["status"] == "completed", status["error"] + def _as_list(value): + return json.loads(value) if isinstance(value, str) else value + + rule_linked = _as_list(session.execute(text( + "SELECT linked_entities FROM logic_rules WHERE ontology_id = 'o-llm' AND name_cn = '设备指纹黑名单拒绝'" + )).scalar_one()) + assert rule_linked == ["设备指纹"] + action_linked = _as_list(session.execute(text( + "SELECT linked_entities FROM actions WHERE ontology_id = 'o-llm' AND name_cn = '标记欺诈'" + )).scalar_one()) + assert action_linked == ["设备指纹"] + finally: + session.close() From 81b319f43bc48a7763c4723b3e33efd5c7a0f793 Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Tue, 15 Sep 2026 15:08:25 +0800 Subject: [PATCH 03/27] feat(extraction): chunk long documents at section boundaries before extraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit graph-engineering playbook scaling guidance: a document large enough that a single extraction call would have to hold the whole thing gets split at markdown heading boundaries (falling back to paragraph boundaries for headerless documents) into <=12000-char chunks, each carrying the tail of the previous chunk as overlap so a concept mentioned right at a section break isn't separated from the relations described just after it. Reuses the existing cross-file _merge_extraction_results/resolve_entities pipeline unchanged for cross-chunk merging — a chunk is just another unit fed into the same executor loop that already handles multiple files, so no new merge logic was needed. Today's real documents (1.3-5.9KB) are all well under the 12000-char threshold and pass through as a single chunk, unchanged from current behavior; this is forward infrastructure for genuinely long documents. --- backend/app/tasks/extraction.py | 58 +++++++++++++++++++++-- backend/tests/test_extraction_chunking.py | 50 +++++++++++++++++++ 2 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 backend/tests/test_extraction_chunking.py diff --git a/backend/app/tasks/extraction.py b/backend/app/tasks/extraction.py index bc6bf7cb..4f4e5153 100644 --- a/backend/app/tasks/extraction.py +++ b/backend/app/tasks/extraction.py @@ -1,6 +1,7 @@ from app.tasks.celery_app import celery_app import logging +import re logger = logging.getLogger(__name__) @@ -249,6 +250,48 @@ def _fuzzy_resolve_entity(name: str, name_to_id: dict) -> str | None: return candidates[0][1] +# ── Long-document chunking (graph-engineering playbook, scaling guidance) ─── +_CHUNK_MAX_CHARS = 12000 +_CHUNK_OVERLAP_CHARS = 500 +_SECTION_HEADER = re.compile(r"^#{1,3}\s+.*$", re.MULTILINE) + + +def _split_sections(md: str) -> list[str]: + """Split markdown into section chunks at heading boundaries. Falls back to + paragraph boundaries when the document has no headings at all.""" + starts = [m.start() for m in _SECTION_HEADER.finditer(md)] + if not starts or starts[0] != 0: + starts = [0] + starts + if len(starts) > 1: + return [md[s:e] for s, e in zip(starts, starts[1:] + [len(md)]) if md[s:e].strip()] + return [p for p in md.split("\n\n") if p.strip()] + + +def _chunk_document(md: str, max_chars: int = _CHUNK_MAX_CHARS, overlap_chars: int = _CHUNK_OVERLAP_CHARS) -> list[str]: + """Chunk a long document at section boundaries so a single extraction call + never has to hold the whole thing — entities/relations still land within + the same chunk as their context. Each chunk after the first repeats the + tail of the previous chunk as overlap, so a concept mentioned at a section + boundary isn't split from the relations described just after it. Short + documents (the common case) pass through unchanged as a single chunk.""" + if len(md) <= max_chars: + return [md] + + sections = _split_sections(md) + chunks: list[str] = [] + current = "" + for section in sections: + if current and len(current) + len(section) > max_chars: + chunks.append(current) + overlap = current[-overlap_chars:] if overlap_chars > 0 else "" + current = overlap + section + else: + current += section + if current.strip(): + chunks.append(current) + return chunks or [md] + + @celery_app.task(bind=True) def run_extraction(self, task_id: str): import app.models # noqa: F401 — register all tables for FK resolution @@ -317,14 +360,19 @@ def _clean(text: str) -> str: max_workers = 1 # serial extraction to avoid OOM with large LLM payloads completed = 0 - task.progress = {"stage": f"extracting files 0/{len(valid_mds)} (parallel ×{max_workers})", "pct": 20} + # a long document is chunked at section boundaries first (see + # _chunk_document) — most documents are short and pass through as a + # single chunk, so this is a no-op for today's typical corpus + units = [(f, chunk) for f in valid_mds for chunk in _chunk_document(_clean(f.converted_md or ""))] + total_units = len(units) + + task.progress = {"stage": f"extracting files 0/{total_units} (parallel ×{max_workers})", "pct": 20} db.commit() with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = {} - for f in valid_mds: - md = _clean(f.converted_md or "") - futures[executor.submit(extract_ontology, md, prompt_content, config_dict, model_name)] = f + for f, chunk in units: + futures[executor.submit(extract_ontology, chunk, prompt_content, config_dict, model_name)] = f for future in as_completed(futures): f = futures[future] @@ -335,7 +383,7 @@ def _clean(text: str) -> str: except Exception: logger.exception("extract_ontology failed for file %s", getattr(f, "id", f)) completed += 1 - task.progress = {"stage": f"extracting files {completed}/{len(valid_mds)} (parallel ×{max_workers})", "pct": 20 + 35 * completed // len(valid_mds)} + task.progress = {"stage": f"extracting files {completed}/{total_units} (parallel ×{max_workers})", "pct": 20 + 35 * completed // total_units} db.commit() if all_results: diff --git a/backend/tests/test_extraction_chunking.py b/backend/tests/test_extraction_chunking.py new file mode 100644 index 00000000..1636aeeb --- /dev/null +++ b/backend/tests/test_extraction_chunking.py @@ -0,0 +1,50 @@ +"""Long-document chunking (graph-engineering playbook, scaling guidance): +a document over the size threshold is chunked at section boundaries with a +trailing overlap, so a single extraction call never has to hold the whole +document — short documents (the common case) pass through untouched. +""" +from app.tasks.extraction import _chunk_document, _split_sections + + +def test_short_document_is_not_chunked(): + md = "# 标题\n\n短文档内容。" + assert _chunk_document(md, max_chars=1000) == [md] + + +def test_long_document_splits_at_section_headers(): + md = ( + "# 第一节\n\n" + "甲" * 40 + "\n\n" + "# 第二节\n\n" + "乙" * 40 + "\n\n" + "# 第三节\n\n" + "丙" * 40 + ) + chunks = _chunk_document(md, max_chars=60, overlap_chars=0) + assert len(chunks) > 1 + assert "甲" * 40 in "".join(chunks) + assert "乙" * 40 in "".join(chunks) + assert "丙" * 40 in "".join(chunks) + # each section stays whole within its chunk, never split mid-section + for section in ("# 第一节", "# 第二节", "# 第三节"): + assert sum(section in c for c in chunks) == 1 + + +def test_chunk_boundary_repeats_overlap_from_previous_chunk(): + md = ( + "# 第一节\n\n" + "甲" * 40 + "\n\n" + "# 第二节\n\n" + "乙" * 40 + ) + chunks = _chunk_document(md, max_chars=50, overlap_chars=10) + assert len(chunks) == 2 + # the tail of chunk 1 reappears at the start of chunk 2 + assert chunks[0][-10:] in chunks[1] + + +def test_document_without_headers_falls_back_to_paragraph_splitting(): + md = "\n\n".join(f"段落{i}" * 20 for i in range(5)) + chunks = _chunk_document(md, max_chars=60, overlap_chars=0) + assert len(chunks) > 1 + for i in range(5): + assert sum(f"段落{i}" * 20 in c for c in chunks) == 1 + + +def test_split_sections_single_section_when_no_headers_and_short(): + assert _split_sections("一段没有标题的短文本") == ["一段没有标题的短文本"] From af894216e7e8618d279849d0e8f41334501ebba0 Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Tue, 15 Sep 2026 15:34:22 +0800 Subject: [PATCH 04/27] feat(extraction): auto-select a stronger model for resolution/relation stages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit graph-engineering playbook model-selection guidance: extraction is high- volume and schema-constrained (speed/cost dominate), while resolution and relation inference weigh conflicting evidence (reasoning quality dominates). Extraction always uses the model the user picked, unchanged. For resolution and relation inference, _select_reasoning_model looks for a non-fast sibling already configured on the same ModelConfig.models list when the user's pick matches a known fast/cheap naming pattern (flash, mini, haiku, lite, turbo, nano, fast) — no new user-facing setting, and behavior is identical to today whenever no stronger sibling is configured or the user already picked the strong variant. --- backend/app/tasks/extraction.py | 37 ++++++++++++++++++- .../tests/test_extraction_model_selection.py | 35 ++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 backend/tests/test_extraction_model_selection.py diff --git a/backend/app/tasks/extraction.py b/backend/app/tasks/extraction.py index 4f4e5153..c54cea18 100644 --- a/backend/app/tasks/extraction.py +++ b/backend/app/tasks/extraction.py @@ -250,6 +250,33 @@ def _fuzzy_resolve_entity(name: str, name_to_id: dict) -> str | None: return candidates[0][1] +# ── Tiered model selection (graph-engineering playbook, model selection) ──── +# Extraction is high-volume and schema-constrained — speed/cost dominate, so +# it always uses whatever model the user picked. Resolution and relation +# inference weigh conflicting evidence — reasoning quality dominates — so +# when the user picked an explicitly fast/cheap variant, we look for a +# stronger sibling already configured on the same model config and use that +# instead. No new user-facing setting: this reads only what's already in +# ModelConfig.models. +_FAST_MODEL_HINTS = ("flash", "mini", "haiku", "lite", "turbo", "nano", "fast") + + +def _select_reasoning_model(model_name: str, available_models: list) -> str: + """Pick the model to use for judgment-heavy stages (resolution, relation + inference). Falls back to `model_name` unchanged unless the user's pick + looks like a fast/cheap variant AND a non-fast sibling exists in the same + model config — so behavior is identical to today whenever no better + candidate is configured.""" + if not any(hint in model_name.lower() for hint in _FAST_MODEL_HINTS): + return model_name + for candidate in available_models or []: + if isinstance(candidate, str) and candidate != model_name and not any( + hint in candidate.lower() for hint in _FAST_MODEL_HINTS + ): + return candidate + return model_name + + # ── Long-document chunking (graph-engineering playbook, scaling guidance) ─── _CHUNK_MAX_CHARS = 12000 _CHUNK_OVERLAP_CHARS = 500 @@ -337,6 +364,12 @@ def run_extraction(self, task_id: str): task.status = "failed"; task.error = "Model or prompt not found"; db.commit(); return model_name = task.parameters.get("model_name", "") + # resolution/relation-inference weigh conflicting evidence — use a + # stronger sibling model already configured on this model config when + # the user picked a fast/cheap variant for extraction (see + # _select_reasoning_model); falls back to model_name unchanged + # otherwise, so this is a no-op unless a better candidate exists + reasoning_model_name = _select_reasoning_model(model_name, model_cfg.models) config_dict = { "provider": model_cfg.provider, "api_key": decrypt(model_cfg.api_key_encrypted or ""), @@ -397,7 +430,7 @@ def _clean(text: str) -> str: task.progress = {"stage": "resolving entities", "pct": 58} db.commit() from app.services.llm_service import resolve_entities, apply_entity_resolution - alias_map = resolve_entities(result["entities"], config_dict, model_name) + alias_map = resolve_entities(result["entities"], config_dict, reasoning_model_name) result = apply_entity_resolution(result, alias_map) else: # 全部失败时回退到合并文本单次提取 @@ -446,7 +479,7 @@ def _clean(text: str) -> str: db.commit() extra_rels = infer_relations( entities_extracted, relations_extracted, - combined_text, config_dict, model_name + combined_text, config_dict, reasoning_model_name ) if extra_rels: # Accept relations where both endpoints fuzzy-match a known entity name diff --git a/backend/tests/test_extraction_model_selection.py b/backend/tests/test_extraction_model_selection.py new file mode 100644 index 00000000..efaf68fc --- /dev/null +++ b/backend/tests/test_extraction_model_selection.py @@ -0,0 +1,35 @@ +"""Tiered model selection (graph-engineering playbook, model selection): +resolution/relation-inference weigh conflicting evidence and should use a +stronger sibling model when the user picked a fast/cheap variant for +extraction — with zero new user-facing setting, reading only what's already +configured in ModelConfig.models. +""" +from app.tasks.extraction import _select_reasoning_model + + +def test_upgrades_from_fast_variant_to_stronger_sibling(): + assert _select_reasoning_model( + "deepseek-flash", ["deepseek-v4-pro", "deepseek-flash"] + ) == "deepseek-v4-pro" + + +def test_recognizes_common_fast_naming_conventions(): + assert _select_reasoning_model("gpt-4o-mini", ["gpt-4o", "gpt-4o-mini"]) == "gpt-4o" + assert _select_reasoning_model("claude-haiku", ["claude-sonnet", "claude-haiku"]) == "claude-sonnet" + + +def test_falls_back_to_same_model_when_not_a_fast_variant(): + """The user already picked the strong model — never downgrade it.""" + assert _select_reasoning_model("deepseek-v4-pro", ["deepseek-v4-pro", "deepseek-flash"]) == "deepseek-v4-pro" + + +def test_falls_back_to_same_model_when_no_stronger_sibling_configured(): + assert _select_reasoning_model("deepseek-flash", ["deepseek-flash"]) == "deepseek-flash" + assert _select_reasoning_model("deepseek-flash", []) == "deepseek-flash" + assert _select_reasoning_model("deepseek-flash", None) == "deepseek-flash" + + +def test_does_not_pick_another_fast_variant_as_the_upgrade(): + assert _select_reasoning_model( + "gpt-4o-mini", ["gpt-4o-mini", "claude-haiku"] + ) == "gpt-4o-mini" From 59925cd692fa9adb6392e2180d27272b1f8c8eb3 Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Tue, 15 Sep 2026 15:39:47 +0800 Subject: [PATCH 05/27] feat(extraction): resolve new entities against already-published ones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit graph-engineering playbook incremental-update guidance: resolve_entities only clustered entities within the current run, so re-running extraction on an ontology that already has saved entities could create a duplicate for a surface-form variant of something that already exists (e.g. a new run spells "客户" as "客户方"). resolve_entities gains an optional existing_entities param: entities already saved on the ontology are added to each type's clustering candidates, marked [已存在] in the prompt with an explicit instruction that their name wins the canonical slot — backed by a deterministic override so a re-run can never silently relabel an already-published entity even if the model ignores the instruction. A lone new entity (which previously skipped clustering entirely as "nothing to resolve against") is now still checked when existing entities of its type exist. run_extraction queries the ontology's current entities before resolution and passes them through; no other behavior changes when no existing entities are present (default param, backward compatible with the 3 pre-existing call sites). --- backend/app/services/llm_service.py | 59 +++++++++++++++--- backend/app/tasks/extraction.py | 16 ++++- .../agent/test_extraction_normalization.py | 61 +++++++++++++++++++ .../tests/agent/test_simple_llm_instances.py | 56 ++++++++++++++++- 4 files changed, 181 insertions(+), 11 deletions(-) diff --git a/backend/app/services/llm_service.py b/backend/app/services/llm_service.py index c4d1b904..ff79b635 100644 --- a/backend/app/services/llm_service.py +++ b/backend/app/services/llm_service.py @@ -167,14 +167,25 @@ def _looks_like_enum_value(name: str) -> bool: return bool(_ENUM_VALUE_PATTERN.match(name)) -def resolve_entities(entities: list[dict], model_config: dict, model_name: str) -> dict[str, str]: +def resolve_entities( + entities: list[dict], model_config: dict, model_name: str, + existing_entities: list[dict] | None = None, +) -> dict[str, str]: """Cross-document entity resolution (graph-engineering playbook, resolution stage): within each entity type, cluster surface-form variants of the same real-world concept using the extraction-time descriptions as disambiguation context — catches cases exact-name matching misses (abbreviation vs full name, alias vs formal name across files). Returns an alias map {raw_name_cn: canonical_name_cn} covering every input name; unmatched names - and failed calls fall back to identity so no entity is ever silently lost.""" + and failed calls fall back to identity so no entity is ever silently lost. + + `existing_entities` (playbook incremental-update guidance: resolve new + entities against the existing canonical set, not against each other) are + entities already saved on this ontology from a prior run. They are never + renamed — if a new entity clusters with one of them, that existing + entity's name_cn is forced as the canonical, so re-running extraction + never silently relabels an already-published entity out from under its + existing relations/instances.""" alias_map: dict[str, str] = {} by_type: dict[str, list[dict]] = {} for e in entities: @@ -184,21 +195,43 @@ def resolve_entities(entities: list[dict], model_config: dict, model_name: str) alias_map[name] = name by_type.setdefault(e.get("type") or "", []).append(e) + existing_by_type: dict[str, list[dict]] = {} + existing_names: set[str] = set() + for e in existing_entities or []: + name = e.get("name_cn") + if not name: + continue + existing_names.add(name) + existing_by_type.setdefault(e.get("type") or "", []).append(e) + provider = model_config.get("provider", "openai") api_key = model_config.get("api_key", "") api_base = model_config.get("api_base") for etype, group in by_type.items(): - if len(group) < 2: + existing_group = existing_by_type.get(etype, []) + if len(group) + len(existing_group) < 2: continue entity_list = "\n".join( f"- {e['name_cn']}: {(e.get('description') or '').strip()[:100]}" for e in group ) + if existing_group: + entity_list += "\n" + "\n".join( + f"- [已存在] {e['name_cn']}: {(e.get('description') or '').strip()[:100]}" + for e in existing_group + ) + incremental_rule = ( + "\n标记为 [已存在] 的实体是本体里已经发布的规范实体:如果某个未标记的新实体与其属于" + "同一概念,该簇的 canonical 必须使用 [已存在] 实体的名字(去掉标记),绝不能改名或" + "另选一个新名字。" + if existing_group else "" + ) system_prompt = ( f"你是实体消歧专家。下面是同一类型({etype or '未分类'})从文档中提取的实体," "其中部分可能是同一概念的不同写法(如全称/简称、中英文并列、别名)。请聚类:" "每个输入名称必须出现在且仅出现在一个簇的 aliases 中;确实不同的概念各自单独成簇;" - "结合描述判断,不要仅凭字面相似合并不同概念;canonical 取信息最完整、无歧义的写法。\n\n" + "结合描述判断,不要仅凭字面相似合并不同概念;canonical 取信息最完整、无歧义的写法。" + f"{incremental_rule}\n\n" '只返回 JSON:{"clusters": [{"canonical": "规范名", "aliases": ["写法1", "写法2"]}]}' ) messages = [ @@ -210,15 +243,25 @@ def resolve_entities(entities: list[dict], model_config: dict, model_name: str) except Exception: continue # resolution is best-effort; keep identity mapping on failure group_names = {e["name_cn"] for e in group} + existing_group_names = {e["name_cn"] for e in existing_group} + all_names = group_names | existing_group_names for cluster in (parsed.get("clusters") or []) if isinstance(parsed, dict) else []: if not isinstance(cluster, dict): continue canonical = _clean_entity_name(cluster.get("canonical")) - if not canonical or canonical not in group_names: + aliases = [_clean_entity_name(a) for a in (cluster.get("aliases") or [])] + aliases = [a for a in aliases if a and a in all_names] + # an existing (already-published) entity in this cluster always + # wins the canonical slot, regardless of what the model picked + existing_in_cluster = [a for a in aliases if a in existing_group_names] + ( + [canonical] if canonical in existing_group_names else [] + ) + if existing_in_cluster: + canonical = existing_in_cluster[0] + elif not canonical or canonical not in group_names: continue - for alias in cluster.get("aliases") or []: - alias = _clean_entity_name(alias) - if alias and alias in group_names: + for alias in aliases: + if alias in group_names: # only remap names from this run's own entities alias_map[alias] = canonical return alias_map diff --git a/backend/app/tasks/extraction.py b/backend/app/tasks/extraction.py index c54cea18..87f96598 100644 --- a/backend/app/tasks/extraction.py +++ b/backend/app/tasks/extraction.py @@ -426,11 +426,23 @@ def _clean(text: str) -> str: # so a concept extracted under different surface forms — whether # across files or within one document — would otherwise fracture # into separate nodes - if len(result.get("entities") or []) > 1: + new_entity_count = len(result.get("entities") or []) + # incremental-update guidance: resolve this run's new entities + # against the ontology's already-published entities too, so a + # re-run doesn't create a duplicate for a surface-form variant of + # something that already exists + existing_entities_for_resolution = [ + {"name_cn": e.name_cn, "description": e.description, "type": e.type} + for e in db.query(Entity).filter(Entity.ontology_id == task.ontology_id).all() + ] + if new_entity_count > 1 or (new_entity_count == 1 and existing_entities_for_resolution): task.progress = {"stage": "resolving entities", "pct": 58} db.commit() from app.services.llm_service import resolve_entities, apply_entity_resolution - alias_map = resolve_entities(result["entities"], config_dict, reasoning_model_name) + alias_map = resolve_entities( + result["entities"], config_dict, reasoning_model_name, + existing_entities=existing_entities_for_resolution, + ) result = apply_entity_resolution(result, alias_map) else: # 全部失败时回退到合并文本单次提取 diff --git a/backend/tests/agent/test_extraction_normalization.py b/backend/tests/agent/test_extraction_normalization.py index 917c0a0e..d3c831c5 100644 --- a/backend/tests/agent/test_extraction_normalization.py +++ b/backend/tests/agent/test_extraction_normalization.py @@ -240,6 +240,67 @@ def test_resolve_entities_single_type_member_skips_llm_call(): assert alias_map == {"唯一实体": "唯一实体"} +def test_resolve_entities_anchors_canonical_to_existing_entity(): + """Incremental-update guidance: a new-run surface form that clusters with + an already-published entity must remap to that entity's own name, not to + whatever the model picked as canonical.""" + from app.services import llm_service + original = llm_service._call_llm + + def fake_call(provider, api_key, api_base, model, messages): + assert "[已存在]" in messages[1]["content"] + return '{"clusters": [{"canonical": "客户方", "aliases": ["客户方", "客户"]}]}' + + llm_service._call_llm = fake_call + try: + new_entities = [{"name_cn": "客户方", "type": "Concept", "description": "本轮新提取"}] + existing = [{"name_cn": "客户", "type": "Concept", "description": "已发布的规范实体"}] + alias_map = resolve_entities( + new_entities, {"provider": "openai", "api_key": "x"}, "m", existing_entities=existing, + ) + finally: + llm_service._call_llm = original + # the model picked "客户方" as canonical, but the existing entity "客户" + # must win regardless + assert alias_map["客户方"] == "客户" + + +def test_resolve_entities_single_new_entity_still_checked_against_existing(): + """A single new entity (which would otherwise skip clustering entirely) + must still be checked against existing entities of the same type.""" + from app.services import llm_service + original = llm_service._call_llm + llm_service._call_llm = lambda *a, **k: ( + '{"clusters": [{"canonical": "借款人", "aliases": ["借款方", "借款人"]}]}' + ) + try: + alias_map = resolve_entities( + [{"name_cn": "借款方", "type": "Concept", "description": "本轮新提取"}], + {"provider": "openai", "api_key": "x"}, "m", + existing_entities=[{"name_cn": "借款人", "type": "Concept", "description": "已发布"}], + ) + finally: + llm_service._call_llm = original + assert alias_map["借款方"] == "借款人" + + +def test_resolve_entities_does_not_touch_existing_entities_of_other_types(): + """Existing entities of a different type never enter the same clustering + call — no cross-type contamination.""" + from app.services import llm_service + original = llm_service._call_llm + llm_service._call_llm = lambda *a, **k: (_ for _ in ()).throw(AssertionError("should not be called")) + try: + alias_map = resolve_entities( + [{"name_cn": "唯一实体", "type": "Concept", "description": "d"}], + {"provider": "openai", "api_key": "x"}, "m", + existing_entities=[{"name_cn": "供应商甲", "type": "Supplier", "description": "d"}], + ) + finally: + llm_service._call_llm = original + assert alias_map == {"唯一实体": "唯一实体"} + + def test_apply_entity_resolution_merges_and_remaps_references(): """apply_entity_resolution merges aliased entities (keeping the richer description), remaps relation endpoints, drops self-loops created by the diff --git a/backend/tests/agent/test_simple_llm_instances.py b/backend/tests/agent/test_simple_llm_instances.py index cc2545f9..0b62696a 100644 --- a/backend/tests/agent/test_simple_llm_instances.py +++ b/backend/tests/agent/test_simple_llm_instances.py @@ -187,7 +187,7 @@ def test_single_document_near_duplicate_entities_are_resolved(schema, monkeypatc ) monkeypatch.setattr( "app.services.llm_service.resolve_entities", - lambda entities, config, model_name: { + lambda entities, config, model_name, existing_entities=None: { "产品研发部": "产品研发部", "产品研发部门": "产品研发部", }, @@ -282,3 +282,57 @@ def _as_list(value): assert action_linked == ["设备指纹"] finally: session.close() + + +def test_rerun_resolves_new_surface_form_against_already_published_entity(schema, monkeypatch): + """Incremental-update guidance: re-running extraction on an ontology that + already has a published entity must not create a duplicate when the new + run's LLM output spells the same concept differently — it should resolve + to the existing entity's own name, not the other way around.""" + factory, session = _worker(schema) + task_id_1 = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + + monkeypatch.setattr( + "app.services.llm_service.extract_ontology", + lambda *a, **k: { + "entities": [{"name_cn": "客户", "type": "Concept", "description": "贷款客户", "properties": {}}], + "relations": [], "logic_rules": [], "actions": [], + }, + ) + extraction_task.run_extraction(task_id_1) + + task_id_2 = str(uuid.uuid4()) + session.execute(text( + "INSERT INTO extraction_tasks (id, ontology_id, prompt_id, model_id, status, parameters, progress, error, created_at, updated_at) " + "VALUES (:id, 'o-llm', 'p-1', 'm-1', 'queued', :params, '{}'::json, NULL, now(), now())" + ), {"id": task_id_2, "params": '{"model_name": "mock-extractor", "constraints": []}'}) + session.commit() + + monkeypatch.setattr( + "app.services.llm_service.extract_ontology", + lambda *a, **k: { + "entities": [{"name_cn": "客户方", "type": "Concept", "description": "本轮换了个写法", "properties": {}}], + "relations": [], "logic_rules": [], "actions": [], + }, + ) + monkeypatch.setattr( + "app.services.llm_service._call_llm", + lambda *a, **k: '{"clusters": [{"canonical": "客户方", "aliases": ["客户方", "客户"]}]}', + ) + extraction_task.run_extraction(task_id_2) + + try: + status = session.execute(text( + "SELECT status, error FROM extraction_tasks WHERE id = :id" + ), {"id": task_id_2}).mappings().one() + assert status["status"] == "completed", status["error"] + names = session.execute(text( + "SELECT name_cn FROM entities WHERE ontology_id = 'o-llm'" + )).scalars().all() + # the model's clustering call picked "客户方" as canonical, but the + # already-published "客户" must win — no duplicate, no rename + assert names == ["客户"] + finally: + session.close() From 26a703c7731afc172d87d617685bf47087248328 Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Tue, 15 Sep 2026 15:45:04 +0800 Subject: [PATCH 06/27] feat(extraction): compute graph connectivity diagnostics after each run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit graph-engineering playbook Section V.E: connected components, edge/node density, and hub (highest-degree) entities are structural health signals for whether entity resolution actually worked — a single component means everything reachable; fragmented islands mean surface-form variants that should have merged did not. New app/services/graph_diagnostics.py: pure functions (no DB, no LLM calls), computed over the ontology's FULL saved entity/relation set (not just this run's output) so a stale count from a prior run never masks today's actual state. Stored on extraction_task.validation_report under a new graph_diagnostics key, alongside the existing P0 report — same place, no new endpoint or table. Isolated as its own module rather than folded into extraction.py: it's pure graph theory unrelated to LLM orchestration, and reusable later for an on-demand ontology-detail-page diagnostic outside the extraction pipeline. --- backend/app/services/graph_diagnostics.py | 68 +++++++++++++++++++ backend/app/tasks/extraction.py | 12 ++++ .../tests/agent/test_simple_llm_instances.py | 35 ++++++++++ backend/tests/test_graph_diagnostics.py | 61 +++++++++++++++++ 4 files changed, 176 insertions(+) create mode 100644 backend/app/services/graph_diagnostics.py create mode 100644 backend/tests/test_graph_diagnostics.py diff --git a/backend/app/services/graph_diagnostics.py b/backend/app/services/graph_diagnostics.py new file mode 100644 index 00000000..639ca470 --- /dev/null +++ b/backend/app/services/graph_diagnostics.py @@ -0,0 +1,68 @@ +"""Graph connectivity diagnostics (graph-engineering playbook, Section V.E): +after entity resolution, a few structural signals reveal whether resolution +actually worked. A single connected component means every entity is +reachable from every other — fragmented islands mean surface-form variants +that should have merged did not. The edge/node ratio gives a density read; +the highest-degree entities are the hub nodes a future summarization stage +would prioritize. + +Pure functions only — no database access, no LLM calls — so this is callable +both from the extraction pipeline and, later, on demand for any ontology. +""" + + +def compute_graph_diagnostics(entities: list[dict], edges: list[tuple[str, str]], top_n: int = 5) -> dict: + """`entities`: [{"id": ..., "name_cn": ...}, ...] every entity in the + ontology. `edges`: [(source_id, target_id), ...] every relation, + direction ignored (weak connectivity, per the playbook).""" + ids = [e["id"] for e in entities if e.get("id")] + names = {e["id"]: e.get("name_cn") or e["id"] for e in entities if e.get("id")} + id_set = set(ids) + + adjacency: dict[str, list[str]] = {i: [] for i in ids} + degree: dict[str, int] = {i: 0 for i in ids} + edge_count = 0 + for source, target in edges: + if source not in id_set or target not in id_set: + continue # a dangling reference is a separate concern (P0 validation) + adjacency[source].append(target) + adjacency[target].append(source) + degree[source] += 1 + degree[target] += 1 + edge_count += 1 + + components = _weakly_connected_components(ids, adjacency) + component_sizes = sorted((len(c) for c in components), reverse=True) + isolated = [i for i in ids if degree[i] == 0] + hubs = sorted(ids, key=lambda i: degree[i], reverse=True)[:top_n] + + return { + "entity_count": len(ids), + "edge_count": edge_count, + "edge_node_ratio": round(edge_count / len(ids), 2) if ids else 0.0, + "connected_components": len(components), + "largest_component_size": component_sizes[0] if component_sizes else 0, + "isolated_entity_count": len(isolated), + "isolated_entities": [names[i] for i in isolated[:top_n]], + "hub_entities": [{"name_cn": names[i], "degree": degree[i]} for i in hubs if degree[i] > 0], + } + + +def _weakly_connected_components(ids: list[str], adjacency: dict[str, list[str]]) -> list[set]: + seen: set[str] = set() + components: list[set] = [] + for start in ids: + if start in seen: + continue + component = {start} + stack = [start] + seen.add(start) + while stack: + node = stack.pop() + for neighbor in adjacency.get(node, []): + if neighbor not in seen: + seen.add(neighbor) + component.add(neighbor) + stack.append(neighbor) + components.append(component) + return components diff --git a/backend/app/tasks/extraction.py b/backend/app/tasks/extraction.py index 87f96598..17aea25a 100644 --- a/backend/app/tasks/extraction.py +++ b/backend/app/tasks/extraction.py @@ -771,6 +771,18 @@ def _match_logic_rules(text: str, logic_name_to_id: dict) -> list: if project: project.status = "created" + # graph-engineering playbook diagnostics: computed over the ontology's + # full saved graph (not just this run's output), so a stale connected- + # components count from a prior run never masks today's actual state + from app.services.graph_diagnostics import compute_graph_diagnostics + final_entities = db.query(Entity).filter(Entity.ontology_id == task.ontology_id).all() + final_relations = db.query(Relation).filter(Relation.ontology_id == task.ontology_id).all() + graph_diagnostics = compute_graph_diagnostics( + [{"id": e.id, "name_cn": e.name_cn} for e in final_entities], + [(r.source_entity, r.target_entity) for r in final_relations], + ) + task.validation_report = {**(task.validation_report or {}), "graph_diagnostics": graph_diagnostics} + task.status = "completed" task.progress = {"stage": "done", "pct": 100} db.commit() diff --git a/backend/tests/agent/test_simple_llm_instances.py b/backend/tests/agent/test_simple_llm_instances.py index 0b62696a..ae7e82f7 100644 --- a/backend/tests/agent/test_simple_llm_instances.py +++ b/backend/tests/agent/test_simple_llm_instances.py @@ -336,3 +336,38 @@ def test_rerun_resolves_new_surface_form_against_already_published_entity(schema assert names == ["客户"] finally: session.close() + + +def test_extraction_task_stores_graph_diagnostics(schema, monkeypatch): + """Graph diagnostics (connected components, hub entities) are computed + over the ontology's full saved graph and stored on the task's + validation_report alongside the existing P0 report.""" + factory, session = _worker(schema) + task_id = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + monkeypatch.setattr( + "app.services.llm_service.extract_ontology", + lambda *a, **k: { + "entities": [ + {"name_cn": "借款人", "type": "Concept", "description": "d", "properties": {}}, + {"name_cn": "贷款", "type": "Concept", "description": "d", "properties": {}}, + {"name_cn": "孤立实体", "type": "Concept", "description": "d", "properties": {}}, + ], + "relations": [{"source": "借款人", "target": "贷款", "type": "APPLIES_FOR", "confidence": 0.9}], + "logic_rules": [], "actions": [], + }, + ) + extraction_task.run_extraction(task_id) + try: + report = session.execute(text( + "SELECT validation_report FROM extraction_tasks WHERE id = :id" + ), {"id": task_id}).scalar_one() + diagnostics = json.loads(report)["graph_diagnostics"] if isinstance(report, str) else report["graph_diagnostics"] + assert diagnostics["entity_count"] == 3 + assert diagnostics["edge_count"] == 1 + assert diagnostics["connected_components"] == 2 # {借款人,贷款} + {孤立实体} + assert diagnostics["isolated_entity_count"] == 1 + assert diagnostics["isolated_entities"] == ["孤立实体"] + finally: + session.close() diff --git a/backend/tests/test_graph_diagnostics.py b/backend/tests/test_graph_diagnostics.py new file mode 100644 index 00000000..6d72185b --- /dev/null +++ b/backend/tests/test_graph_diagnostics.py @@ -0,0 +1,61 @@ +"""Graph connectivity diagnostics (graph-engineering playbook, Section V.E): +connected components, edge/node density, and hub nodes as a health signal +for whether entity resolution actually merged what it should have. +""" +from app.services.graph_diagnostics import compute_graph_diagnostics + + +def _entities(*names): + return [{"id": n, "name_cn": n} for n in names] + + +def test_single_connected_component_when_fully_linked(): + entities = _entities("借款人", "贷款", "还款记录") + edges = [("借款人", "贷款"), ("贷款", "还款记录")] + diag = compute_graph_diagnostics(entities, edges) + assert diag["connected_components"] == 1 + assert diag["largest_component_size"] == 3 + assert diag["isolated_entity_count"] == 0 + + +def test_fragmented_islands_are_counted_separately(): + entities = _entities("A", "B", "C", "D") + edges = [("A", "B"), ("C", "D")] + diag = compute_graph_diagnostics(entities, edges) + assert diag["connected_components"] == 2 + assert diag["largest_component_size"] == 2 + + +def test_isolated_entities_are_zero_degree_and_reported(): + entities = _entities("A", "B", "孤立实体") + edges = [("A", "B")] + diag = compute_graph_diagnostics(entities, edges) + assert diag["isolated_entity_count"] == 1 + assert diag["isolated_entities"] == ["孤立实体"] + + +def test_edge_node_ratio_and_hub_ranking(): + entities = _entities("hub", "a", "b", "c") + edges = [("hub", "a"), ("hub", "b"), ("hub", "c")] + diag = compute_graph_diagnostics(entities, edges) + assert diag["entity_count"] == 4 + assert diag["edge_count"] == 3 + assert diag["edge_node_ratio"] == 0.75 + assert diag["hub_entities"][0] == {"name_cn": "hub", "degree": 3} + + +def test_dangling_edge_endpoints_are_ignored(): + """An edge referencing an entity id not in the ontology (a data quality + issue caught elsewhere by P0 validation) must not crash diagnostics.""" + entities = _entities("A", "B") + edges = [("A", "B"), ("A", "不存在的实体")] + diag = compute_graph_diagnostics(entities, edges) + assert diag["edge_count"] == 1 + assert diag["connected_components"] == 1 + + +def test_empty_ontology_does_not_divide_by_zero(): + diag = compute_graph_diagnostics([], []) + assert diag["entity_count"] == 0 + assert diag["edge_node_ratio"] == 0.0 + assert diag["connected_components"] == 0 From f707b72ca9b1a6dec386a4eeab834c45b742933c Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Tue, 15 Sep 2026 18:52:36 +0800 Subject: [PATCH 07/27] feat(extraction): synthesize a profile for hub (high-degree) entities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit graph-engineering playbook Section V.A: an entity with enough graph connectivity (degree >= 3) benefits from cross-relation synthesis — a 2-3 paragraph summary, 3-5 atomic traceable key facts, and a time range, grounded in its own description and its known graph relations (no raw document re-scanning needed; the description and relations are already first-class extracted data). Stored under new summary/key_facts/ time_range/summary_degree keys in Entity.properties — description itself stays untouched since several other places (resolution, disambiguation) depend on it staying a short, one-line grounded string. Re-summarizes only when degree changed since summary_degree was last recorded, so a re-run doesn't re-call the model for an unchanged hub every time. Uses the same reasoning_model_name already selected for resolution/relation-inference — no new model-selection logic needed. Fixes a real bug surfaced while testing the skip behavior: entity upsert was replacing ent.properties outright on every re-run instead of merging, silently destroying a previous run's summary (or any other property a later run's extraction didn't re-emit). --- backend/app/services/llm_service.py | 45 +++++++++ backend/app/tasks/extraction.py | 34 ++++++- .../agent/test_extraction_normalization.py | 44 +++++++++ .../tests/agent/test_simple_llm_instances.py | 98 +++++++++++++++++++ 4 files changed, 220 insertions(+), 1 deletion(-) diff --git a/backend/app/services/llm_service.py b/backend/app/services/llm_service.py index ff79b635..5575ad32 100644 --- a/backend/app/services/llm_service.py +++ b/backend/app/services/llm_service.py @@ -389,6 +389,51 @@ def infer_relations(entities: list, existing_relations: list, text: str, return [] # relation inference failure is non-fatal +def summarize_entity( + name: str, description: str, relations_text: str, model_config: dict, model_name: str, +) -> dict | None: + """Hub-node profile synthesis (graph-engineering playbook, Section V.A): + for a high-degree entity, synthesize a 2-3 paragraph summary, 3-5 atomic + traceable key facts, and a time range from its grounded description and + its known graph relations. Returns None on failure — summarization is + best-effort and never blocks extraction.""" + provider = model_config.get("provider", "openai") + api_key = model_config.get("api_key", "") + api_base = model_config.get("api_base") + + system_prompt = ( + f"为本体实体「{name}」生成知识图谱画像。\n\n" + f"已知描述:{description or '(无)'}\n\n" + f"该实体在图中的已知关系:\n{relations_text or '(无)'}\n\n" + "基于以上信息写一段2-3段的事实性综合描述;如信息有冲突,优先采用更具体的说法;" + "提炼3-5条可溯源到上述描述/关系的原子关键事实;不要编造描述和关系之外没有支持的事实;" + "时间范围用 YYYY 或 YYYY-MM 格式,没有明确时间信息就填 \"unknown\"。\n\n" + '只返回 JSON:{"summary": "...", "key_facts": ["...", "..."], ' + '"time_range": {"start": "...", "end": "..."}}' + ) + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": f"实体:{name}"}, + ] + try: + parsed = _parse_response(_call_llm(provider, api_key, api_base, model_name, messages)) + except Exception: + return None + if not isinstance(parsed, dict): + return None + time_range = parsed.get("time_range") + if not isinstance(time_range, dict): + time_range = {"start": "unknown", "end": "unknown"} + return { + "summary": str(parsed.get("summary") or ""), + "key_facts": [f for f in (parsed.get("key_facts") or []) if isinstance(f, str)][:5], + "time_range": { + "start": str(time_range.get("start") or "unknown"), + "end": str(time_range.get("end") or "unknown"), + }, + } + + def _call_llm(provider: str, api_key: str, api_base: str | None, model: str, messages: list, json_mode: bool = True) -> str: # Stable seed for reproducibility: derived from message content so same input → same seed. import hashlib as _hashlib, json as _json diff --git a/backend/app/tasks/extraction.py b/backend/app/tasks/extraction.py index 17aea25a..ea04f64c 100644 --- a/backend/app/tasks/extraction.py +++ b/backend/app/tasks/extraction.py @@ -540,7 +540,10 @@ def _clean(text: str) -> str: ent = existing_ent_map[name_cn] # Always allow LLM to enrich description, properties, name_en, name_abbr if e_data.get("description"): ent.description = e_data["description"] - if props: ent.properties = props + # merge, don't replace — a prior run's hub summary + # (summary/key_facts/summary_degree) must survive a re-run + # that doesn't touch this entity's properties otherwise + if props: ent.properties = {**(ent.properties or {}), **props} if e_data.get("name_en"): ent.name_en = e_data["name_en"] if name_abbr: ent.name_abbr = name_abbr if e_data.get("type"): ent.type = e_data["type"] @@ -783,6 +786,35 @@ def _match_logic_rules(text: str, logic_name_to_id: dict) -> list: ) task.validation_report = {**(task.validation_report or {}), "graph_diagnostics": graph_diagnostics} + # hub-node summarization (playbook Section V.A): only entities with + # enough graph connectivity to benefit from cross-relation synthesis; + # re-summarizes only when degree changed since the last summary, so a + # re-run doesn't re-call the model for an unchanged hub every time + _HUB_DEGREE_THRESHOLD = 3 + degree_by_id: dict = {e.id: 0 for e in final_entities} + relation_lines_by_id: dict = {} + name_by_id = {e.id: e.name_cn for e in final_entities} + for r in final_relations: + if r.source_entity in degree_by_id: + degree_by_id[r.source_entity] += 1 + if r.target_entity in degree_by_id: + degree_by_id[r.target_entity] += 1 + line = f"({name_by_id.get(r.source_entity, r.source_entity)}) --[{r.type}]--> ({name_by_id.get(r.target_entity, r.target_entity)})" + relation_lines_by_id.setdefault(r.source_entity, []).append(line) + relation_lines_by_id.setdefault(r.target_entity, []).append(line) + + from app.services.llm_service import summarize_entity + for e in final_entities: + degree = degree_by_id.get(e.id, 0) + if degree < _HUB_DEGREE_THRESHOLD: + continue + if (e.properties or {}).get("summary_degree") == degree: + continue # graph neighborhood unchanged since last summary + relations_text = "\n".join(sorted(set(relation_lines_by_id.get(e.id, [])))) + profile = summarize_entity(e.name_cn, e.description or "", relations_text, config_dict, reasoning_model_name) + if profile: + e.properties = {**(e.properties or {}), **profile, "summary_degree": degree} + task.status = "completed" task.progress = {"stage": "done", "pct": 100} db.commit() diff --git a/backend/tests/agent/test_extraction_normalization.py b/backend/tests/agent/test_extraction_normalization.py index d3c831c5..0edae9a7 100644 --- a/backend/tests/agent/test_extraction_normalization.py +++ b/backend/tests/agent/test_extraction_normalization.py @@ -21,6 +21,7 @@ extract_ontology, normalize_extracted_ontology, resolve_entities, + summarize_entity, ) BACKEND_DIR = Path(__file__).resolve().parents[2] @@ -348,3 +349,46 @@ def test_apply_entity_resolution_is_noop_for_identity_map(): result = {"entities": [{"name_cn": "A"}], "relations": []} assert apply_entity_resolution(result, {"A": "A"}) is result assert apply_entity_resolution(result, {}) is result + + +def test_summarize_entity_parses_profile_from_llm_response(): + from app.services import llm_service + original = llm_service._call_llm + llm_service._call_llm = lambda *a, **k: ( + '{"summary": "借款人是信贷业务的核心概念。", ' + '"key_facts": ["借款人需通过授信审批", "借款人拥有贷款账户", "多出来的第四条", ' + '"第五条", "第六条会被截断"], ' + '"time_range": {"start": "2026", "end": "ongoing"}}' + ) + try: + profile = summarize_entity( + "借款人", "贷款申请人", "(借款人) --[APPLIES_FOR]--> (贷款)", + {"provider": "openai", "api_key": "x"}, "m", + ) + finally: + llm_service._call_llm = original + assert profile["summary"] == "借款人是信贷业务的核心概念。" + assert len(profile["key_facts"]) == 5 # capped at 5 + assert profile["time_range"] == {"start": "2026", "end": "ongoing"} + + +def test_summarize_entity_returns_none_on_failure(): + from app.services import llm_service + original = llm_service._call_llm + llm_service._call_llm = lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom")) + try: + profile = summarize_entity("借款人", "d", "", {"provider": "openai", "api_key": "x"}, "m") + finally: + llm_service._call_llm = original + assert profile is None + + +def test_summarize_entity_defaults_missing_time_range_to_unknown(): + from app.services import llm_service + original = llm_service._call_llm + llm_service._call_llm = lambda *a, **k: '{"summary": "s", "key_facts": []}' + try: + profile = summarize_entity("X", "d", "", {"provider": "openai", "api_key": "x"}, "m") + finally: + llm_service._call_llm = original + assert profile["time_range"] == {"start": "unknown", "end": "unknown"} diff --git a/backend/tests/agent/test_simple_llm_instances.py b/backend/tests/agent/test_simple_llm_instances.py index ae7e82f7..4f90e280 100644 --- a/backend/tests/agent/test_simple_llm_instances.py +++ b/backend/tests/agent/test_simple_llm_instances.py @@ -371,3 +371,101 @@ def test_extraction_task_stores_graph_diagnostics(schema, monkeypatch): assert diagnostics["isolated_entities"] == ["孤立实体"] finally: session.close() + + +_HUB_RESULT = { + "entities": [ + {"name_cn": "借款人", "type": "Concept", "description": "贷款申请人", "properties": {}}, + {"name_cn": "贷款", "type": "Concept", "description": "d", "properties": {}}, + {"name_cn": "授信额度", "type": "Concept", "description": "d", "properties": {}}, + {"name_cn": "信用评分", "type": "Concept", "description": "d", "properties": {}}, + ], + "relations": [ + {"source": "借款人", "target": "贷款", "type": "APPLIES_FOR", "confidence": 0.9}, + {"source": "借款人", "target": "授信额度", "type": "HAS", "confidence": 0.9}, + {"source": "借款人", "target": "信用评分", "type": "HAS", "confidence": 0.9}, + ], + "logic_rules": [], "actions": [], +} + + +def test_hub_entity_gets_summarized(schema, monkeypatch): + """A degree-3+ entity (playbook Section V.A hub threshold) gets a + synthesized profile stored on its properties; low-degree entities do not.""" + factory, session = _worker(schema) + task_id = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: _HUB_RESULT) + monkeypatch.setattr( + "app.services.llm_service._call_llm", + lambda *a, **k: '{"summary": "借款人是核心概念。", "key_facts": ["f1"], ' + '"time_range": {"start": "2026", "end": "ongoing"}}', + ) + extraction_task.run_extraction(task_id) + try: + status = session.execute(text( + "SELECT status, error FROM extraction_tasks WHERE id = :id" + ), {"id": task_id}).mappings().one() + assert status["status"] == "completed", status["error"] + rows = session.execute(text( + "SELECT name_cn, properties FROM entities WHERE ontology_id = 'o-llm'" + )).mappings().all() + props_by_name = {r["name_cn"]: (json.loads(r["properties"]) if isinstance(r["properties"], str) else r["properties"]) for r in rows} + assert props_by_name["借款人"]["summary"] == "借款人是核心概念。" + assert props_by_name["借款人"]["summary_degree"] == 3 + # a degree-1 entity is not summarized + assert "summary" not in props_by_name["贷款"] + finally: + session.close() + + +def test_hub_entity_summary_skipped_when_degree_unchanged_on_rerun(schema, monkeypatch): + """Re-running with the same graph shape must not re-call the summarizer — + only a changed source-document set (reflected here as a changed degree) + should trigger re-summarization.""" + factory, session = _worker(schema) + task_id_1 = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: _HUB_RESULT) + # resolve_entities isn't exercised on a from-scratch run (nothing existing + # to resolve against yet), so mocking summarize_entity alone is enough + # here and avoids also stubbing out the (unrelated) resolution LLM call + monkeypatch.setattr( + "app.services.llm_service.summarize_entity", + lambda *a, **k: {"summary": "first pass", "key_facts": [], "time_range": {"start": "unknown", "end": "unknown"}}, + ) + extraction_task.run_extraction(task_id_1) + + task_id_2 = str(uuid.uuid4()) + session.execute(text( + "INSERT INTO extraction_tasks (id, ontology_id, prompt_id, model_id, status, parameters, progress, error, created_at, updated_at) " + "VALUES (:id, 'o-llm', 'p-1', 'm-1', 'queued', :params, '{}'::json, NULL, now(), now())" + ), {"id": task_id_2, "params": '{"model_name": "mock-extractor", "constraints": []}'}) + session.commit() + + monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: _HUB_RESULT) + # resolve_entities still runs (4 new entities of the same type) and needs + # a harmless response so it doesn't attempt a real network call; the + # assertion this test cares about is that summarize_entity is skipped + monkeypatch.setattr("app.services.llm_service._call_llm", lambda *a, **k: '{"clusters": []}') + monkeypatch.setattr( + "app.services.llm_service.summarize_entity", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("summarizer should not be called again")), + ) + extraction_task.run_extraction(task_id_2) + + try: + status = session.execute(text( + "SELECT status, error FROM extraction_tasks WHERE id = :id" + ), {"id": task_id_2}).mappings().one() + assert status["status"] == "completed", status["error"] + properties = session.execute(text( + "SELECT properties FROM entities WHERE ontology_id = 'o-llm' AND name_cn = '借款人'" + )).scalar_one() + properties = json.loads(properties) if isinstance(properties, str) else properties + # untouched from the first run — the summarizer was never invoked again + assert properties["summary"] == "first pass" + finally: + session.close() From db90e31252a158f01b80ae5be745af8eefdf75fc Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Tue, 15 Sep 2026 21:46:53 +0800 Subject: [PATCH 08/27] feat(evals): add real precision/recall eval, verify against 4 domains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds precision_recall() to the document-extraction eval harness, using fuzzy (rapidfuzz) greedy one-to-one matching against a gold entity list per case — a real quality signal instead of the existing loose keyword-presence check. Wired additively into run.py (a domain without a *.entities.json gold file simply skips it, no breakage). Verified end-to-end with a live DeepSeek extraction run against real test_data documents across 信贷/财务/供应链/医疗 (per explicit request to cover all four, not just 信贷). Results (F1: 0.58/0.00/0.38/0.38) show the harness correctly surfaces two distinct gap types: real missed concepts (e.g. ACEI/ARB, 董事会, 审计委员会) and naming- granularity mismatches where the extractor emits a more specific compound name than the gold concept (财务's 0.0 case: 采购审批权限 vs 采购审批, 预算管理 vs 预算) — both are genuine signal, not tuned away. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01AkAraQUPvCkrAKpk89TMNR --- ...233\345\272\224\351\223\276.entities.json" | 17 +++++ .../\344\277\241\350\264\267.entities.json" | 24 +++++++ .../\345\214\273\347\226\227.entities.json" | 19 ++++++ .../\350\264\242\345\212\241.entities.json" | 17 +++++ .../manifest/4domains.json | 6 ++ backend/evals/document_extraction/run.py | 19 +++++- .../evals/document_extraction/validators.py | 45 ++++++++++++++ .../test_document_extraction_validators.py | 62 ++++++++++++++++++- 8 files changed, 207 insertions(+), 2 deletions(-) create mode 100644 "backend/evals/document_extraction/ground_truth/\344\276\233\345\272\224\351\223\276.entities.json" create mode 100644 "backend/evals/document_extraction/ground_truth/\344\277\241\350\264\267.entities.json" create mode 100644 "backend/evals/document_extraction/ground_truth/\345\214\273\347\226\227.entities.json" create mode 100644 "backend/evals/document_extraction/ground_truth/\350\264\242\345\212\241.entities.json" create mode 100644 backend/evals/document_extraction/manifest/4domains.json diff --git "a/backend/evals/document_extraction/ground_truth/\344\276\233\345\272\224\351\223\276.entities.json" "b/backend/evals/document_extraction/ground_truth/\344\276\233\345\272\224\351\223\276.entities.json" new file mode 100644 index 00000000..c992b477 --- /dev/null +++ "b/backend/evals/document_extraction/ground_truth/\344\276\233\345\272\224\351\223\276.entities.json" @@ -0,0 +1,17 @@ +[ + "供应商", + "供应商分级", + "库存", + "物料", + "安全库存", + "采购申请", + "采购审批", + "质量控制", + "来料检验", + "供应商审计", + "物流", + "配送", + "承运商", + "质检部", + "董事会" +] diff --git "a/backend/evals/document_extraction/ground_truth/\344\277\241\350\264\267.entities.json" "b/backend/evals/document_extraction/ground_truth/\344\277\241\350\264\267.entities.json" new file mode 100644 index 00000000..d0709fdb --- /dev/null +++ "b/backend/evals/document_extraction/ground_truth/\344\277\241\350\264\267.entities.json" @@ -0,0 +1,24 @@ +[ + "借款人", + "金融机构资金方", + "信贷科技平台", + "信贷驱动服务", + "平台服务", + "重资本模式", + "轻资本模式", + "信用分层", + "信用评分", + "授信额度", + "授信审批", + "逾期阶段", + "贷后催收", + "风险准备金", + "获客渠道", + "转化漏斗", + "风险定价模型", + "客户生命周期", + "合规部", + "风控委员会", + "反欺诈黑名单", + "多头借贷指标" +] diff --git "a/backend/evals/document_extraction/ground_truth/\345\214\273\347\226\227.entities.json" "b/backend/evals/document_extraction/ground_truth/\345\214\273\347\226\227.entities.json" new file mode 100644 index 00000000..cb492fcb --- /dev/null +++ "b/backend/evals/document_extraction/ground_truth/\345\214\273\347\226\227.entities.json" @@ -0,0 +1,19 @@ +[ + "高血压", + "2型糖尿病", + "收缩压", + "舒张压", + "空腹血糖", + "糖化血红蛋白", + "肾功能", + "ACEI", + "ARB", + "CCB", + "二甲双胍", + "胰岛素", + "糖尿病肾病", + "视网膜病变", + "低血糖", + "药物相互作用", + "用药重整" +] diff --git "a/backend/evals/document_extraction/ground_truth/\350\264\242\345\212\241.entities.json" "b/backend/evals/document_extraction/ground_truth/\350\264\242\345\212\241.entities.json" new file mode 100644 index 00000000..2827b0ca --- /dev/null +++ "b/backend/evals/document_extraction/ground_truth/\350\264\242\345\212\241.entities.json" @@ -0,0 +1,17 @@ +[ + "供应商", + "发票", + "采购订单", + "采购审批", + "付款", + "费用报销", + "预算", + "预算调整", + "应收账款", + "账龄", + "坏账准备", + "固定资产", + "资产折旧", + "审计委员会", + "董事会" +] diff --git a/backend/evals/document_extraction/manifest/4domains.json b/backend/evals/document_extraction/manifest/4domains.json new file mode 100644 index 00000000..a6353945 --- /dev/null +++ b/backend/evals/document_extraction/manifest/4domains.json @@ -0,0 +1,6 @@ +[ + {"case_id": "credit-01", "domain": "信贷", "source_file": "test_data/信贷/信贷业务战略.md", "ground_truth_id": "信贷"}, + {"case_id": "supply-chain-01", "domain": "供应链", "source_file": "test_data/供应链/supply_chain_strategy.md", "ground_truth_id": "供应链"}, + {"case_id": "medical-01", "domain": "医疗", "source_file": "test_data/医疗/clinical_protocols.md", "ground_truth_id": "医疗"}, + {"case_id": "finance-01", "domain": "财务", "source_file": "test_data/财务/financial_controls.md", "ground_truth_id": "财务"} +] diff --git a/backend/evals/document_extraction/run.py b/backend/evals/document_extraction/run.py index 4674df8d..6fd5f73b 100644 --- a/backend/evals/document_extraction/run.py +++ b/backend/evals/document_extraction/run.py @@ -4,7 +4,9 @@ import json import os -from evals.document_extraction.validators import dedup_gate, instance_leakage_gate, keyword_recall, rule_leakage_gate +from evals.document_extraction.validators import ( + dedup_gate, instance_leakage_gate, keyword_recall, precision_recall, rule_leakage_gate, +) def run_eval(manifest_path: str, ground_truth_dir: str, adapter, repo_root: str) -> dict: @@ -17,6 +19,15 @@ def run_eval(manifest_path: str, ground_truth_dir: str, adapter, repo_root: str) with open(gt_path, encoding="utf-8") as f: ground_truth = json.load(f) + # optional: a full gold entity list for real precision/recall/F1, + # separate from the keyword-hint ground_truth above — a domain + # without this file simply skips precision/recall, no breakage + entities_gt_path = os.path.join(ground_truth_dir, f"{case['ground_truth_id']}.entities.json") + gold_entities = None + if os.path.exists(entities_gt_path): + with open(entities_gt_path, encoding="utf-8") as f: + gold_entities = json.load(f) + source_path = os.path.join(repo_root, case["source_file"]) try: extraction_result = adapter.extract(source_path) @@ -28,6 +39,8 @@ def run_eval(manifest_path: str, ground_truth_dir: str, adapter, repo_root: str) "leakage_findings": instance_leakage_gate(extraction_result), "rule_leakage_findings": rule_leakage_gate(extraction_result), } + if gold_entities is not None: + case_entry["precision_recall"] = precision_recall(gold_entities, extraction_result) except Exception as e: case_entry = { "case_id": case["case_id"], @@ -42,12 +55,16 @@ def run_eval(manifest_path: str, ground_truth_dir: str, adapter, repo_root: str) sum(c["keyword_recall"]["score"] for c in recall_cases) / len(recall_cases) if recall_cases else 0.0 ) + pr_cases = [c for c in case_results if "precision_recall" in c] + mean_f1 = sum(c["precision_recall"]["f1"] for c in pr_cases) / len(pr_cases) if pr_cases else None summary = { "total_cases": total, "mean_keyword_recall": mean_recall, "cases_with_dedup_findings": sum(1 for c in case_results if c.get("dedup_findings")), "cases_with_leakage_findings": sum(1 for c in case_results if c.get("leakage_findings")), "cases_with_rule_leakage_findings": sum(1 for c in case_results if c.get("rule_leakage_findings")), + "cases_with_precision_recall": len(pr_cases), + "mean_f1": mean_f1, } return {"cases": case_results, "summary": summary} diff --git a/backend/evals/document_extraction/validators.py b/backend/evals/document_extraction/validators.py index 91079689..2e545313 100644 --- a/backend/evals/document_extraction/validators.py +++ b/backend/evals/document_extraction/validators.py @@ -55,6 +55,51 @@ def keyword_recall(ground_truth: list[dict], extraction_result: dict) -> dict: return {"score": score, "passed": passed, "failed": failed} +def precision_recall(gold_entities: list[str], extraction_result: dict, fuzzy_threshold: int = 85) -> dict: + """Real precision/recall/F1 against a full gold-labeled entity list (not + keyword_recall's loose "does this keyword appear somewhere" check). + Fuzzy-matched (rapidfuzz, same threshold convention as dedup_gate) so a + minor surface-form difference between the gold name and the extracted + name_cn doesn't count as a miss — each gold entity and each extracted + entity is consumed by at most one match, so duplicates on either side + can't inflate the score.""" + extracted_names = [ + str(item.get("name_cn") or "") for item in (extraction_result.get("entities") or []) + if isinstance(item, dict) and item.get("name_cn") + ] + unmatched_extracted = list(extracted_names) + matched_gold: list[str] = [] + missed_gold: list[str] = [] + for gold_name in gold_entities: + best_idx, best_score = None, -1 + for idx, candidate in enumerate(unmatched_extracted): + score = fuzz.ratio(gold_name, candidate) + if score > best_score: + best_idx, best_score = idx, score + if best_idx is not None and best_score >= fuzzy_threshold: + matched_gold.append(gold_name) + unmatched_extracted.pop(best_idx) + else: + missed_gold.append(gold_name) + + true_positive = len(matched_gold) + false_negative = len(missed_gold) + false_positive = len(unmatched_extracted) # extracted but never matched a gold entity + precision = true_positive / (true_positive + false_positive) if (true_positive + false_positive) else 1.0 + recall = true_positive / (true_positive + false_negative) if (true_positive + false_negative) else 1.0 + f1 = (2 * precision * recall / (precision + recall)) if (precision + recall) else 0.0 + return { + "precision": round(precision, 3), + "recall": round(recall, 3), + "f1": round(f1, 3), + "true_positive": true_positive, + "false_positive": false_positive, + "false_negative": false_negative, + "missed_gold_entities": missed_gold, + "unmatched_extracted_entities": unmatched_extracted, + } + + def dedup_gate(extraction_result: dict, threshold: int = 85) -> list[dict]: findings: list[dict] = [] for category in ("entities", "logic_rules", "actions"): diff --git a/backend/tests/evals/test_document_extraction_validators.py b/backend/tests/evals/test_document_extraction_validators.py index acc813ee..4229f6c0 100644 --- a/backend/tests/evals/test_document_extraction_validators.py +++ b/backend/tests/evals/test_document_extraction_validators.py @@ -3,7 +3,9 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from evals.document_extraction.validators import keyword_recall, dedup_gate, instance_leakage_gate, rule_leakage_gate +from evals.document_extraction.validators import ( + keyword_recall, dedup_gate, instance_leakage_gate, rule_leakage_gate, precision_recall, +) SAMPLE_RESULT = { "entities": [ @@ -141,3 +143,61 @@ def test_rule_leakage_gate_clean_for_concept_names(): "relations": [], "logic_rules": [], "actions": [], "instances": [], } assert rule_leakage_gate(result) == [] + + +def test_precision_recall_perfect_match(): + gold = ["借款人", "信用分层"] + result = {"entities": [{"name_cn": "借款人"}, {"name_cn": "信用分层"}]} + pr = precision_recall(gold, result) + assert pr["precision"] == 1.0 + assert pr["recall"] == 1.0 + assert pr["f1"] == 1.0 + assert pr["false_positive"] == 0 + assert pr["false_negative"] == 0 + + +def test_precision_recall_counts_extra_extracted_entities_as_false_positives(): + gold = ["借款人"] + result = {"entities": [{"name_cn": "借款人"}, {"name_cn": "促销定价规则"}]} + pr = precision_recall(gold, result) + assert pr["true_positive"] == 1 + assert pr["false_positive"] == 1 + assert pr["recall"] == 1.0 + assert pr["precision"] == 0.5 + assert pr["unmatched_extracted_entities"] == ["促销定价规则"] + + +def test_precision_recall_counts_missing_gold_entities_as_false_negatives(): + gold = ["借款人", "授信额度"] + result = {"entities": [{"name_cn": "借款人"}]} + pr = precision_recall(gold, result) + assert pr["true_positive"] == 1 + assert pr["false_negative"] == 1 + assert pr["precision"] == 1.0 + assert pr["recall"] == 0.5 + assert pr["missed_gold_entities"] == ["授信额度"] + + +def test_precision_recall_fuzzy_matches_minor_surface_form_differences(): + gold = ["产品研发部"] + result = {"entities": [{"name_cn": "产品研发部门"}]} + pr = precision_recall(gold, result) + assert pr["true_positive"] == 1 + assert pr["missed_gold_entities"] == [] + + +def test_precision_recall_does_not_double_count_one_extracted_entity_for_two_gold_entities(): + """Each extracted entity can satisfy at most one gold entity — a single + lucky match must not inflate recall for an unrelated second gold entity.""" + gold = ["借款人", "还款人"] + result = {"entities": [{"name_cn": "借款人"}]} + pr = precision_recall(gold, result) + assert pr["true_positive"] == 1 + assert pr["false_negative"] == 1 + + +def test_precision_recall_empty_gold_is_perfect_precision_no_recall_penalty(): + pr = precision_recall([], {"entities": [{"name_cn": "任意实体"}]}) + assert pr["recall"] == 1.0 # nothing was required + assert pr["false_positive"] == 1 + assert pr["precision"] == 0.0 From dbc6e2219de35b83b4189269de80b6c3658ea57f Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Tue, 15 Sep 2026 23:08:14 +0800 Subject: [PATCH 09/27] fix(extraction): flush session before computing graph diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit app/database.py's SessionLocal is autoflush=False, but the graph- diagnostics and hub-summarization block queried Entity/Relation back from the session right after adding this run's own new rows, with no db.flush() in between — so the query only saw whatever was committed before this run started, missing everything just added. Caught via a live Playwright-driven rebuild of the 信贷/财务/供应链/ 医疗 simple-LLM ontologies: a brand-new 医疗 ontology's diagnostics showed edge_count=0 despite 101 relations landing in the table moments later. The existing unit tests never caught this because their _worker fixture's session factory defaults to autoflush=True, unlike production — added a dedicated regression test that builds its session with autoflush=False to match app/database.py and reproduces the bug (verified it fails without the db.flush() fix, passes with it). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01AkAraQUPvCkrAKpk89TMNR --- backend/app/tasks/extraction.py | 5 ++- .../tests/agent/test_simple_llm_instances.py | 33 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/backend/app/tasks/extraction.py b/backend/app/tasks/extraction.py index ea04f64c..a5bccee7 100644 --- a/backend/app/tasks/extraction.py +++ b/backend/app/tasks/extraction.py @@ -776,7 +776,10 @@ def _match_logic_rules(text: str, logic_name_to_id: dict) -> list: # graph-engineering playbook diagnostics: computed over the ontology's # full saved graph (not just this run's output), so a stale connected- - # components count from a prior run never masks today's actual state + # components count from a prior run never masks today's actual state. + # session autoflush is off (app/database.py), so this run's own + # entity/relation adds above must be flushed before querying them back. + db.flush() from app.services.graph_diagnostics import compute_graph_diagnostics final_entities = db.query(Entity).filter(Entity.ontology_id == task.ontology_id).all() final_relations = db.query(Relation).filter(Relation.ontology_id == task.ontology_id).all() diff --git a/backend/tests/agent/test_simple_llm_instances.py b/backend/tests/agent/test_simple_llm_instances.py index 4f90e280..e7327a88 100644 --- a/backend/tests/agent/test_simple_llm_instances.py +++ b/backend/tests/agent/test_simple_llm_instances.py @@ -373,6 +373,39 @@ def test_extraction_task_stores_graph_diagnostics(schema, monkeypatch): session.close() +def test_graph_diagnostics_sees_this_runs_own_relations_under_autoflush_off(schema, monkeypatch): + """Production's SessionLocal is autoflush=False (app/database.py), unlike + the _worker fixture's default-autoflush factory used above. The + diagnostics query must still see this run's own newly-added entities and + relations rather than only whatever was committed before this run began.""" + factory = sessionmaker(bind=create_engine(_scoped_url(schema)), autoflush=False) + session = factory() + task_id = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + monkeypatch.setattr( + "app.services.llm_service.extract_ontology", + lambda *a, **k: { + "entities": [ + {"name_cn": "借款人", "type": "Concept", "description": "d", "properties": {}}, + {"name_cn": "贷款", "type": "Concept", "description": "d", "properties": {}}, + ], + "relations": [{"source": "借款人", "target": "贷款", "type": "APPLIES_FOR", "confidence": 0.9}], + "logic_rules": [], "actions": [], + }, + ) + extraction_task.run_extraction(task_id) + try: + report = session.execute(text( + "SELECT validation_report FROM extraction_tasks WHERE id = :id" + ), {"id": task_id}).scalar_one() + diagnostics = json.loads(report)["graph_diagnostics"] if isinstance(report, str) else report["graph_diagnostics"] + assert diagnostics["edge_count"] == 1 + assert diagnostics["connected_components"] == 1 + finally: + session.close() + + _HUB_RESULT = { "entities": [ {"name_cn": "借款人", "type": "Concept", "description": "贷款申请人", "properties": {}}, From f95fa382bc0793a7e06b6a02238161f7a58de8ee Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Wed, 16 Sep 2026 14:32:27 +0800 Subject: [PATCH 10/27] feat(extraction): support instance-level relations, fix medical concept/instance boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an instance_relations output channel to the shared concept/instance directive so a causal/interaction relation between two SPECIFIC named things (e.g. 2型糖尿病 causes 糖尿病肾病, 阿司匹林 interacts_with 华法林) survives even though relations only connect concept-level entities. Reuses the existing but previously-unwired EntityInstanceRelation model (an edge between two EntityInstance rows, anchored to the concept-level Relation it instantiates) — no migration needed. Neo4j sync now also pushes instance nodes and instance-relation edges, labeled distinctly from their concept nodes. Root cause of the reported bug: _merge_extraction_results rebuilds a fresh dict from the per-chunk LLM outputs and was silently dropping any key it didn't already know about, so instance_relations never survived past the merge step even once emitted. Also needed an explicit db.flush() between the auto-created concept Relation and the EntityInstanceRelation that references it — autoflush's cross-table ordering wasn't reliable here despite the declared FK, so plausible-looking automatic ordering was made explicit instead of trusted. Also updates the 医疗本体提取 prompt (DB-stored, not in git — this is a manual content edit, not a code change) to drop a legacy "each document must produce one type=Disease entity named after the filename" instruction that predates the concept/instance discipline and directly contradicted it — specific diseases/drugs now correctly go to instances via entity_type, with the disease-drug relation graph preserved through the new instance_relations channel instead of being silently discarded. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01AkAraQUPvCkrAKpk89TMNR --- backend/app/tasks/extraction.py | 117 +++++++++++++++++- .../tests/agent/test_simple_llm_instances.py | 100 +++++++++++++++ 2 files changed, 216 insertions(+), 1 deletion(-) diff --git a/backend/app/tasks/extraction.py b/backend/app/tasks/extraction.py index a5bccee7..01ddbac7 100644 --- a/backend/app/tasks/extraction.py +++ b/backend/app/tasks/extraction.py @@ -9,6 +9,7 @@ from app.models import ( # noqa: E402, F401 user, ontology, file, prompt, model_config, entity, logic as logic_model, action, relation, extraction_task, rules_config, + entity_instance, entity_instance_relation, ) # ── 概念/实例分离指令 — 追加到所有提取 Prompt 之后 ─────────────────────────── @@ -23,6 +24,11 @@ "instances": [{"entity_type": "所属概念的name_cn", "name_cn": "实例名称", "name_en": "可选", "properties": {"属性1": "值"}, "confidence": 0.9}] relations 只在概念之间建立(如 供应商 -[supply]-> 产品),不要在具体实例之间建立关系。 +如果两个具体命名实例之间存在文档明确写出的因果/治疗/相互作用等关系(如"2型糖尿病 导致 糖尿病肾病""阿司匹林 与 华法林 相互作用"), +放入新增的 instance_relations 数组,不要因为"relations 只在概念之间建立"而丢弃这类信息: +"instance_relations": [{"source": "实例名称", "target": "实例名称", "type": "causes|interacts_with|治疗|等英文或语义明确的关系类型", "confidence": 0.9}] +source/target 必须是 instances 数组里已出现的 name_cn,不要引用未提取的实例或概念实体本身。 + 属性的枚举值/分级/分档/阶段标签(如信用分层的"A层""B层""C层"、逾期阶段的"M0""M1""M2+"、 融资轮次的"A轮融资""D轮融资")不是独立的概念,也不是需要单独建 instance 的命名对象——它们是所属 概念(如"信用分层""逾期阶段""融资")某个属性的可选取值。不要为它们创建 entity,也不要创建 @@ -40,6 +46,7 @@ def _merge_extraction_results(results: list[dict]) -> dict: all_logic: list[dict] = [] all_actions: list[dict] = [] all_instances: list[dict] = [] + all_instance_relations: list[dict] = [] # 收集全部 for r in results: @@ -62,6 +69,9 @@ def _merge_extraction_results(results: list[dict]) -> dict: for inst in (r.get("instances") or []): if isinstance(inst, dict): all_instances.append(inst) + for ir in (r.get("instance_relations") or []): + if isinstance(ir, dict): + all_instance_relations.append(ir) # 实体去重(按 name_cn,保留属性最丰富的) seen_entities: dict[str, dict] = {} @@ -125,12 +135,22 @@ def _merge_extraction_results(results: list[dict]) -> dict: seen_instances.add(key) instances.append(inst) + # 实例关系去重(source-target-type 唯一) + seen_inst_rels: set = set() + instance_relations = [] + for ir in all_instance_relations: + key = (ir.get("source", ""), ir.get("target", ""), ir.get("type", "")) + if key not in seen_inst_rels and key[0] and key[1]: + seen_inst_rels.add(key) + instance_relations.append(ir) + return { "entities": entities, "relations": relations, "logic_rules": list(seen_logic.values()), "actions": list(seen_actions.values()), "instances": instances, + "instance_relations": instance_relations, } @@ -576,6 +596,10 @@ def _clean(text: str) -> str: # ── 写入实例数据(EntityInstance 表)— 挂在概念实体下,对齐 Pipeline Mapping ── from app.models.entity_instance import EntityInstance import hashlib as _hl + # instance_relations (below) needs to resolve a source/target instance + # name back to its stable id and owning concept entity id + instance_name_to_id: dict = {} + instance_id_to_entity_id: dict = {} for inst_data in result.get("instances", []): if not isinstance(inst_data, dict): continue @@ -610,11 +634,14 @@ def _clean(text: str) -> str: id=stable_id, entity_id=concept_id, ontology_id=task.ontology_id, row_identity=inst_name, row_data=inst_props, )) + instance_name_to_id[inst_name] = stable_id + instance_id_to_entity_id[stable_id] = concept_id db.commit() # ── Fix 2+4: upsert relations (by source_id, target_id, type) ──────── existing_rels = db.query(Relation).filter(Relation.ontology_id == task.ontology_id).all() existing_rel_set = {(r.source_entity, r.target_entity, r.type) for r in existing_rels} + relation_id_by_key: dict = {(r.source_entity, r.target_entity, r.type): r.id for r in existing_rels} for rel in result.get("relations", []): src_name = rel.get("source") or rel.get("source_entity", "") @@ -626,12 +653,65 @@ def _clean(text: str) -> str: if rel_type in ("关联", "未知", "相关", "其他", "") or not rel_type.isascii(): continue if src_id and tgt_id and (src_id, tgt_id, rel_type) not in existing_rel_set: + new_rel_id = str(uuid.uuid4()) db.add(Relation( - id=str(uuid.uuid4()), ontology_id=task.ontology_id, + id=new_rel_id, ontology_id=task.ontology_id, source_entity=src_id, target_entity=tgt_id, type=rel_type, confidence=rel.get("confidence", 0.85), )) existing_rel_set.add((src_id, tgt_id, rel_type)) + relation_id_by_key[(src_id, tgt_id, rel_type)] = new_rel_id + + # ── instance-level relations (graph-engineering playbook): specific + # named things (e.g. 2型糖尿病, 阿司匹林) keep their own relation graph + # via EntityInstanceRelation, anchored to the concept-level Relation + # it instantiates (auto-created here if the LLM didn't also emit it) ── + from app.models.entity_instance_relation import EntityInstanceRelation + existing_inst_rels = db.query(EntityInstanceRelation).filter( + EntityInstanceRelation.ontology_id == task.ontology_id).all() + existing_inst_rel_set = { + (r.source_instance_id, r.target_instance_id, r.relation_definition_id) + for r in existing_inst_rels + } + + for ir in result.get("instance_relations", []): + if not isinstance(ir, dict): + continue + src_iid = _fuzzy_resolve_entity(ir.get("source", ""), instance_name_to_id) + tgt_iid = _fuzzy_resolve_entity(ir.get("target", ""), instance_name_to_id) + if not src_iid or not tgt_iid: + continue + ir_type = (ir.get("type") or "RELATED").strip() + if ir_type in ("关联", "未知", "相关", "其他", "") or not ir_type.isascii(): + continue + src_concept_id = instance_id_to_entity_id.get(src_iid) + tgt_concept_id = instance_id_to_entity_id.get(tgt_iid) + if not src_concept_id or not tgt_concept_id: + continue + concept_key = (src_concept_id, tgt_concept_id, ir_type) + rel_def_id = relation_id_by_key.get(concept_key) + if not rel_def_id: + rel_def_id = str(uuid.uuid4()) + db.add(Relation( + id=rel_def_id, ontology_id=task.ontology_id, + source_entity=src_concept_id, target_entity=tgt_concept_id, + type=ir_type, confidence=ir.get("confidence", 0.8), + )) + existing_rel_set.add(concept_key) + relation_id_by_key[concept_key] = rel_def_id + # the FK from entity_instance_relations to relations must see + # this row on insert — flush now rather than trust autoflush + # ordering across the two tables + db.flush() + edge_key = (src_iid, tgt_iid, rel_def_id) + if edge_key not in existing_inst_rel_set: + db.add(EntityInstanceRelation( + id=str(uuid.uuid4()), ontology_id=task.ontology_id, + source_instance_id=src_iid, target_instance_id=tgt_iid, + relation_definition_id=rel_def_id, + properties={"confidence": ir.get("confidence", 0.8)}, + )) + existing_inst_rel_set.add(edge_key) # ── Keyword matching helpers (unchanged) ───────────────────────────── all_entity_names = [ @@ -857,6 +937,8 @@ def _sync_neo4j(db, ontology_id: str) -> None: from app.services.v2.graph.neo4j_service import Neo4jService from app.models.entity import Entity from app.models.relation import Relation + from app.models.entity_instance import EntityInstance + from app.models.entity_instance_relation import EntityInstanceRelation svc = Neo4jService() if not svc.available: @@ -904,6 +986,39 @@ def _sync_neo4j(db, ontology_id: str) -> None: props={"id": r.id, "ontology_id": ontology_id, "confidence": r.confidence or 0.85}, ) + # 具体命名实例(如 2型糖尿病、阿司匹林)作为独立节点同步,标签在概念类型 + # 后加 Instance 后缀,与概念节点区分开,避免图查询把概念和实例混为一谈 + instances = db.query(EntityInstance).filter(EntityInstance.ontology_id == ontology_id).all() + instance_label_map: dict[str, str] = {} + for inst in instances: + concept_type = entity_type_map.get(inst.entity_id, "OntologyEntity") + label = f"{concept_type}Instance" + instance_label_map[inst.id] = label + props = { + **(inst.row_data or {}), + "id": inst.id, + "ontology_id": ontology_id, + "entity_id": inst.entity_id, + "name_cn": inst.row_identity, + } + svc.upsert_entity(label, props, key_field="id") + + # 实例级关系(语义概念层面的 relation 已在上面写入;具体实例之间的边只 + # 存在于 Neo4j,不进 Postgres relations 表) + inst_relations = db.query(EntityInstanceRelation).filter( + EntityInstanceRelation.ontology_id == ontology_id).all() + relation_type_by_id = {r.id: r.type for r in relations} + for ir in inst_relations: + src_label = instance_label_map.get(ir.source_instance_id, "OntologyEntityInstance") + tgt_label = instance_label_map.get(ir.target_instance_id, "OntologyEntityInstance") + rel_type = (relation_type_by_id.get(ir.relation_definition_id) or "RELATED").upper().replace(" ", "_").replace("-", "_") + svc.upsert_relation( + src_label=src_label, src_key=ir.source_instance_id, + tgt_label=tgt_label, tgt_key=ir.target_instance_id, + rel_type=rel_type, + props={"id": ir.id, "ontology_id": ontology_id, **(ir.properties or {})}, + ) + svc.close() except Exception: logger.warning("Neo4j sync failed for ontology %s; extraction result unaffected", ontology_id, exc_info=True) diff --git a/backend/tests/agent/test_simple_llm_instances.py b/backend/tests/agent/test_simple_llm_instances.py index e7327a88..4a37bcc8 100644 --- a/backend/tests/agent/test_simple_llm_instances.py +++ b/backend/tests/agent/test_simple_llm_instances.py @@ -163,6 +163,106 @@ def test_simple_llm_tabular_extraction_materializes_instances(schema, monkeypatc session.close() +def test_instance_relations_create_concept_relation_and_instance_edge(schema, monkeypatch): + """A causal/interaction relation between two SPECIFIC named instances + (e.g. 2型糖尿病 causes 糖尿病肾病) must not be silently dropped just + because 'relations only connect concepts' — it auto-creates the concept- + level Relation it instantiates and lands as an EntityInstanceRelation + edge between the two instance rows.""" + factory, session = _worker(schema) + task_id = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + + fake_result = { + "entities": [{"name_cn": "疾病", "type": "Disease", "description": "d", "properties": {}}], + "relations": [], "logic_rules": [], "actions": [], + "instances": [ + {"entity_type": "疾病", "name_cn": "2型糖尿病", "properties": {}}, + {"entity_type": "疾病", "name_cn": "糖尿病肾病", "properties": {}}, + ], + "instance_relations": [ + {"source": "2型糖尿病", "target": "糖尿病肾病", "type": "causes", "confidence": 0.9}, + ], + } + monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: fake_result) + + extraction_task.run_extraction(task_id) + try: + status = session.execute(text( + "SELECT status, error FROM extraction_tasks WHERE id = :id" + ), {"id": task_id}).mappings().one() + assert status["status"] == "completed", status["error"] + + concept_rels = session.execute(text( + "SELECT type FROM relations WHERE ontology_id = 'o-llm'" + )).mappings().all() + assert [r["type"] for r in concept_rels] == ["causes"] + + edge = session.execute(text( + "SELECT ei_src.row_identity AS src, ei_tgt.row_identity AS tgt " + "FROM entity_instance_relations eir " + "JOIN entity_instances ei_src ON ei_src.id = eir.source_instance_id " + "JOIN entity_instances ei_tgt ON ei_tgt.id = eir.target_instance_id " + "WHERE eir.ontology_id = 'o-llm'" + )).mappings().one() + assert edge["src"] == "2型糖尿病" + assert edge["tgt"] == "糖尿病肾病" + finally: + session.close() + + +def test_instance_relations_rerun_does_not_duplicate(schema, monkeypatch): + """Re-running extraction with the same instance_relations output must not + create a second EntityInstanceRelation edge or a second concept Relation.""" + factory, session = _worker(schema) + task_id_1 = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + + fake_result = { + "entities": [{"name_cn": "疾病", "type": "Disease", "description": "d", "properties": {}}], + "relations": [], "logic_rules": [], "actions": [], + "instances": [ + {"entity_type": "疾病", "name_cn": "2型糖尿病", "properties": {}}, + {"entity_type": "疾病", "name_cn": "糖尿病肾病", "properties": {}}, + ], + "instance_relations": [ + {"source": "2型糖尿病", "target": "糖尿病肾病", "type": "causes", "confidence": 0.9}, + ], + } + monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: fake_result) + extraction_task.run_extraction(task_id_1) + + task_id_2 = str(uuid.uuid4()) + session.execute(text( + "INSERT INTO extraction_tasks (id, ontology_id, prompt_id, model_id, status, parameters, progress, error, created_at, updated_at) " + "VALUES (:id, 'o-llm', 'p-1', 'm-1', 'queued', :params, '{}'::json, NULL, now(), now())" + ), {"id": task_id_2, "params": '{"model_name": "mock-extractor", "constraints": []}'}) + session.commit() + monkeypatch.setattr( + "app.services.llm_service._call_llm", + lambda *a, **k: '{"clusters": []}', + ) + extraction_task.run_extraction(task_id_2) + + try: + status = session.execute(text( + "SELECT status, error FROM extraction_tasks WHERE id = :id" + ), {"id": task_id_2}).mappings().one() + assert status["status"] == "completed", status["error"] + rel_count = session.execute(text( + "SELECT count(*) FROM relations WHERE ontology_id = 'o-llm'" + )).scalar_one() + edge_count = session.execute(text( + "SELECT count(*) FROM entity_instance_relations WHERE ontology_id = 'o-llm'" + )).scalar_one() + assert rel_count == 1 + assert edge_count == 1 + finally: + session.close() + + def test_single_document_near_duplicate_entities_are_resolved(schema, monkeypatch): """Sub-project E3 bug 1: a single-file extraction with near-duplicate entity names (e.g. "产品研发部" vs "产品研发部门") must still go through From 44f8fe71e516486992864a2c190fef3fd6b9214e Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Wed, 16 Sep 2026 17:12:25 +0800 Subject: [PATCH 11/27] fix(extraction): flush concept relations before instance_relations reuses them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Caught live on a real 供应链 rebuild: FK violation inserting into entity_instance_relations because relation_definition_id pointed at a Relation row that was still only pending in the session, not yet flushed to Postgres. Root cause: the earlier db.flush() fix only covered relations newly created *within* the instance_relations loop itself. It missed the case where a concept-level relation comes from the plain `relations` array (the ordinary relations-upsert loop that runs just before) and gets reused via relation_id_by_key by an instance_relations entry — that relation was never flushed, so it doesn't exist yet when EntityInstanceRelation's FK insert runs. Also fixed the same test-environment gap as before: the two existing instance_relations tests used the _worker fixture's default-autoflush session, which silently flushes pending relations and therefore never exercises this bug. Switched them (and the new regression test) to an explicit autoflush=False session matching production's SessionLocal. Also adds a domain-specific instance_relations example to the 财务, 供应链, and 通用本体提取 prompts (the last one is what 信贷 actually runs under, since it has no dedicated prompt) — the medical prompt got this treatment already; these three only had the shared directive's single abstract example, which was clearly getting less attention in their long, rule-heavy prompts than in medical's causal prose. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01AkAraQUPvCkrAKpk89TMNR --- backend/app/tasks/extraction.py | 5 ++ .../tests/agent/test_simple_llm_instances.py | 66 +++++++++++++++++-- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/backend/app/tasks/extraction.py b/backend/app/tasks/extraction.py index 01ddbac7..c5d8fd99 100644 --- a/backend/app/tasks/extraction.py +++ b/backend/app/tasks/extraction.py @@ -662,6 +662,11 @@ def _clean(text: str) -> str: existing_rel_set.add((src_id, tgt_id, rel_type)) relation_id_by_key[(src_id, tgt_id, rel_type)] = new_rel_id + # instance_relations below may reuse any of the relations just added + # via relation_id_by_key as their relation_definition_id FK — flush + # now so those rows exist before EntityInstanceRelation references them + db.flush() + # ── instance-level relations (graph-engineering playbook): specific # named things (e.g. 2型糖尿病, 阿司匹林) keep their own relation graph # via EntityInstanceRelation, anchored to the concept-level Relation diff --git a/backend/tests/agent/test_simple_llm_instances.py b/backend/tests/agent/test_simple_llm_instances.py index 4a37bcc8..f5d66094 100644 --- a/backend/tests/agent/test_simple_llm_instances.py +++ b/backend/tests/agent/test_simple_llm_instances.py @@ -163,13 +163,69 @@ def test_simple_llm_tabular_extraction_materializes_instances(schema, monkeypatc session.close() +def test_instance_relations_reuses_relation_from_plain_relations_array(schema, monkeypatch): + """Bug caught live on a 供应链 rebuild: when the LLM's own `relations` array + (not the instance_relations auto-create branch) is what creates the + concept-level Relation that an instance_relations entry then reuses via + relation_id_by_key, that relation must still be flushed before the + EntityInstanceRelation insert — otherwise it's an unflushed pending row + and the FK insert fails. Production's SessionLocal is autoflush=False + (app/database.py), unlike _worker's default-autoflush factory, so this + test builds its own session to match and actually catch the bug.""" + factory = sessionmaker(bind=create_engine(_scoped_url(schema)), autoflush=False) + session = factory() + task_id = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + + fake_result = { + "entities": [ + {"name_cn": "供应商", "type": "Supplier", "description": "d", "properties": {}}, + {"name_cn": "物料", "type": "Material", "description": "d", "properties": {}}, + ], + "relations": [ + {"source": "供应商", "target": "物料", "type": "supply", "confidence": 0.9}, + ], + "logic_rules": [], "actions": [], + "instances": [ + {"entity_type": "供应商", "name_cn": "天钢原材料有限公司", "properties": {}}, + {"entity_type": "物料", "name_cn": "钢材", "properties": {}}, + ], + "instance_relations": [ + {"source": "天钢原材料有限公司", "target": "钢材", "type": "supply", "confidence": 0.9}, + ], + } + monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: fake_result) + + extraction_task.run_extraction(task_id) + try: + status = session.execute(text( + "SELECT status, error FROM extraction_tasks WHERE id = :id" + ), {"id": task_id}).mappings().one() + assert status["status"] == "completed", status["error"] + + edge = session.execute(text( + "SELECT ei_src.row_identity AS src, ei_tgt.row_identity AS tgt " + "FROM entity_instance_relations eir " + "JOIN entity_instances ei_src ON ei_src.id = eir.source_instance_id " + "JOIN entity_instances ei_tgt ON ei_tgt.id = eir.target_instance_id " + "WHERE eir.ontology_id = 'o-llm'" + )).mappings().one() + assert edge["src"] == "天钢原材料有限公司" + assert edge["tgt"] == "钢材" + finally: + session.close() + + def test_instance_relations_create_concept_relation_and_instance_edge(schema, monkeypatch): """A causal/interaction relation between two SPECIFIC named instances (e.g. 2型糖尿病 causes 糖尿病肾病) must not be silently dropped just because 'relations only connect concepts' — it auto-creates the concept- level Relation it instantiates and lands as an EntityInstanceRelation - edge between the two instance rows.""" - factory, session = _worker(schema) + edge between the two instance rows. Uses an explicit autoflush=False + session to match production (app/database.py), not _worker's default.""" + factory = sessionmaker(bind=create_engine(_scoped_url(schema)), autoflush=False) + session = factory() task_id = _seed(session) monkeypatch.setattr("app.database.SessionLocal", factory) import app.tasks.extraction as extraction_task @@ -214,8 +270,10 @@ def test_instance_relations_create_concept_relation_and_instance_edge(schema, mo def test_instance_relations_rerun_does_not_duplicate(schema, monkeypatch): """Re-running extraction with the same instance_relations output must not - create a second EntityInstanceRelation edge or a second concept Relation.""" - factory, session = _worker(schema) + create a second EntityInstanceRelation edge or a second concept Relation. + Uses an explicit autoflush=False session to match production.""" + factory = sessionmaker(bind=create_engine(_scoped_url(schema)), autoflush=False) + session = factory() task_id_1 = _seed(session) monkeypatch.setattr("app.database.SessionLocal", factory) import app.tasks.extraction as extraction_task From 405c4853ec9fda765243c1ea6034625ed08a9d6c Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Wed, 16 Sep 2026 23:03:35 +0800 Subject: [PATCH 12/27] modify models.py and extraction.py, fix bugs in models --- backend/app/routers/models.py | 18 +++++++++++++++++- backend/app/tasks/extraction.py | 9 +++++++++ backend/app/tasks/topology.py | 22 ++++++++++++++++++++++ frontend/src/i18n/en.json | 2 ++ frontend/src/i18n/zh.json | 2 ++ frontend/src/pages/models/ModelsPage.tsx | 13 ++++++++++--- 6 files changed, 62 insertions(+), 4 deletions(-) diff --git a/backend/app/routers/models.py b/backend/app/routers/models.py index 58227054..5749c667 100644 --- a/backend/app/routers/models.py +++ b/backend/app/routers/models.py @@ -325,14 +325,30 @@ def delete_model(model_id: str, db: Session = Depends(get_db), _=Depends(require c = db.query(ModelConfig).filter(ModelConfig.id == model_id).first() if not c: raise HTTPException(404, "Not found") + # Block only on a REAL dependent — an Agent version pinned to one of this + # model's versions, or a prompt-generation provenance row — not merely on + # the model's own version-1 row that create_model() always bootstraps. + # Checking "any model_config_versions row exists" made every model saved + # through the normal create flow permanently undeletable. referenced = db.execute(text( - "SELECT count(*) FROM model_config_versions WHERE model_config_id = :id" + "SELECT count(*) FROM model_config_versions v WHERE v.model_config_id = :id AND (" + "EXISTS (SELECT 1 FROM agent_versions av WHERE av.default_model_config_version_id = v.id) " + "OR EXISTS (SELECT 1 FROM prompt_generations pg WHERE pg.model_config_version_id = v.id))" ), {"id": model_id}).scalar_one() if referenced > 0: raise HTTPException(409, detail="MODEL_REFERENCED") db.query(ExtractionTask).filter(ExtractionTask.model_id == model_id).update( {ExtractionTask.model_id: None}, synchronize_session=False ) + # model_config_versions/model_credentials/model_migration_findings all + # RESTRICT-reference model_configs.id, so they must be cleared first — + # and model_configs.active_version_id RESTRICT-references the version + # row itself (the reverse direction), so that pointer must be nulled + # before the version row can go. + db.execute(text("UPDATE model_configs SET active_version_id = NULL WHERE id = :id"), {"id": model_id}) + db.execute(text("DELETE FROM model_credentials WHERE model_config_id = :id"), {"id": model_id}) + db.execute(text("DELETE FROM model_migration_findings WHERE model_config_id = :id"), {"id": model_id}) + db.execute(text("DELETE FROM model_config_versions WHERE model_config_id = :id"), {"id": model_id}) db.delete(c) db.commit() diff --git a/backend/app/tasks/extraction.py b/backend/app/tasks/extraction.py index c5d8fd99..35a8f52e 100644 --- a/backend/app/tasks/extraction.py +++ b/backend/app/tasks/extraction.py @@ -918,6 +918,12 @@ def _match_logic_rules(text: str, logic_name_to_id: dict) -> list: if task: task.status = "failed" task.error = str(e) + # the router set the Ontology to "creating" when this task was + # queued — on failure it must go back to "draft" or it stays + # stuck showing "creating" forever with no task left to finish it + project = db.query(OntologyProject).filter(OntologyProject.id == task.ontology_id).first() + if project and project.status == "creating": + project.status = "draft" db.commit() except Exception: # session 可能已损坏,用新 session 兜底标记失败,避免任务永远卡在 running @@ -928,6 +934,9 @@ def _match_logic_rules(text: str, logic_name_to_id: dict) -> list: if task: task.status = "failed" task.error = str(e) + project = fresh_db.query(OntologyProject).filter(OntologyProject.id == task.ontology_id).first() + if project and project.status == "creating": + project.status = "draft" fresh_db.commit() fresh_db.close() except Exception: diff --git a/backend/app/tasks/topology.py b/backend/app/tasks/topology.py index e30f1fa3..9f6bcdef 100644 --- a/backend/app/tasks/topology.py +++ b/backend/app/tasks/topology.py @@ -76,6 +76,19 @@ "agent.retention_purge": {"queue": QUEUE_HOUSEKEEPING}, } +# These four share one queue/worker pool. Without acks_late + a time limit, a +# single hung LLM call (or a worker dying mid-task) permanently wedges one of +# `--concurrency` slots with nothing to free it — every later task on this +# queue then sits "queued" forever with no worker left to pick it up. +ARTIFACT_EXTRACTION_TASK_NAMES = ( + "app.tasks.extraction.run_extraction", + "app.tasks.audit.run_audit", + "app.tasks.v2.mapping_apply.mapping_apply_task", + "app.tasks.v2.pipeline_run.pipeline_run_task", +) +ARTIFACT_EXTRACTION_SOFT_TIME_LIMIT_SECONDS = 1800 +ARTIFACT_EXTRACTION_HARD_TIME_LIMIT_SECONDS = 1860 + # ── Bounded refresh worker settings ───────────────────────────────────── @dataclass(frozen=True) @@ -227,3 +240,12 @@ def configure_celery_topology(celery_app: Celery) -> None: } for name in REFRESH_TASK_NAMES } + celery_app.conf.task_annotations.update({ + name: { + "acks_late": True, + "reject_on_worker_lost": True, + "soft_time_limit": ARTIFACT_EXTRACTION_SOFT_TIME_LIMIT_SECONDS, + "time_limit": ARTIFACT_EXTRACTION_HARD_TIME_LIMIT_SECONDS, + } + for name in ARTIFACT_EXTRACTION_TASK_NAMES + }) diff --git a/frontend/src/i18n/en.json b/frontend/src/i18n/en.json index 6ab298cf..686d39c7 100644 --- a/frontend/src/i18n/en.json +++ b/frontend/src/i18n/en.json @@ -328,6 +328,8 @@ "empty": "No models configured. Click \"Add Model\" to get started.", "confirm_delete": "Confirm Delete", "confirm_delete_msg": "Delete model \"{{name}}\"?", + "delete_failed_referenced": "Cannot delete: this model is still used by an Agent. Unbind or delete the Agent first.", + "delete_failed_generic": "Delete failed, please try again", "config_type_llm": "LLM Config", "config_type_ocr": "OCR Config", "config_type_other": "Other Config", diff --git a/frontend/src/i18n/zh.json b/frontend/src/i18n/zh.json index ad9a4afe..c5a393ed 100644 --- a/frontend/src/i18n/zh.json +++ b/frontend/src/i18n/zh.json @@ -328,6 +328,8 @@ "empty": "暂无模型配置,点击「添加模型」开始", "confirm_delete": "确认删除", "confirm_delete_msg": "删除模型「{{name}}」?", + "delete_failed_referenced": "无法删除:该模型仍被智能体使用,请先解绑或删除相关智能体后再删除", + "delete_failed_generic": "删除失败,请稍后重试", "config_type_llm": "LLM配置", "config_type_ocr": "OCR配置", "config_type_other": "其他配置", diff --git a/frontend/src/pages/models/ModelsPage.tsx b/frontend/src/pages/models/ModelsPage.tsx index c720ddce..2f70202a 100644 --- a/frontend/src/pages/models/ModelsPage.tsx +++ b/frontend/src/pages/models/ModelsPage.tsx @@ -91,6 +91,7 @@ export default function ModelsPage() { const qc = useQueryClient() const [showCreate, setShowCreate] = useState(false) const [deleteTarget, setDeleteTarget] = useState(null) + const [deleteError, setDeleteError] = useState('') const [testResult, setTestResult] = useState>({}) const [formTags, setFormTags] = useState([]) const { register, handleSubmit, reset, setValue: setCreateValue, control } = useForm({ @@ -114,7 +115,11 @@ export default function ModelsPage() { const deleteMut = useMutation({ mutationFn: (id: string) => modelApi.delete(id), - onSuccess: () => { qc.invalidateQueries({ queryKey: ['models'] }); setDeleteTarget(null) }, + onSuccess: () => { qc.invalidateQueries({ queryKey: ['models'] }); setDeleteTarget(null); setDeleteError('') }, + onError: (err: unknown) => { + const detail = (err as { detail?: string })?.detail + setDeleteError(detail === 'MODEL_REFERENCED' ? t('model.delete_failed_referenced') : t('model.delete_failed_generic')) + }, }) const testMut = useMutation({ @@ -197,7 +202,7 @@ export default function ModelsPage() {
- +
@@ -274,7 +279,9 @@ export default function ModelsPage() { )} deleteTarget && deleteMut.mutate(deleteTarget.id)} onCancel={() => setDeleteTarget(null)} /> + error={deleteError} + onConfirm={() => deleteTarget && deleteMut.mutate(deleteTarget.id)} + onCancel={() => { setDeleteTarget(null); setDeleteError('') }} /> ) } From dd10d6dcfb115987a15c0de4b9fd50f64d336172 Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Thu, 17 Sep 2026 15:08:24 +0800 Subject: [PATCH 13/27] fix(delete): sweep outbox re-inserts, clean up Neo4j/Chroma/MinIO on delete MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs reported by community feedback, both verified by direct investigation before fixing: 1. An ontology with real instance data could never be deleted: the cleanup loop deleted agent_index_outbox before entity_instance_relations, but the latter's AFTER DELETE trigger re-emits fresh outbox rows — left uncleaned, they blocked the ontology row's own delete with ONTOLOGY_HAS_DEPENDENT_DATA. Reordered so agent_index_outbox is cleaned last, sweeping up anything the instance-table triggers just recreated. 2. Deleting an ontology or a curated dataset never touched Neo4j, Chroma, or MinIO, even though delete methods already existed for all three (Neo4jService.delete_by_ontology, ChromaService. delete_collection, StorageService.delete_object) — simply never called from any delete path, leaving permanent orphaned data. Ontology delete now best-effort cleans up its Neo4j nodes/relations and its `ontology_{id}` Chroma collection; curated dataset delete now best-effort removes each MediaItem's storage_uri/ocr_result_uri MinIO object. Both are wrapped so an unreachable external service never blocks or fails an already-committed Postgres delete. Regression tests: a live-Postgres test reproduces the outbox trigger bug end to end (fails on the old cleanup order, passes on the new one); mocked-service tests confirm the external cleanup calls fire with the right ontology/collection id and that failures there don't prevent the 204 response. Two other reported issues (permanently-immutable published-ontology releases, and the resulting misleading delete error message) are a product/compliance decision about audit-immutability boundaries, not addressed here. A third (legacy v2_curated_datasets table not cleaned up by curated delete) needs its own migration-status investigation, also not addressed here. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01AkAraQUPvCkrAKpk89TMNR --- backend/app/routers/ontologies.py | 25 ++++- backend/app/routers/v2/curated.py | 16 ++++ backend/tests/test_ontologies.py | 48 ++++++++++ .../test_ontology_deletion_with_instances.py | 95 +++++++++++++++++++ backend/tests/v2/test_curated_deletion.py | 78 +++++++++++++++ 5 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 backend/tests/test_ontology_deletion_with_instances.py create mode 100644 backend/tests/v2/test_curated_deletion.py diff --git a/backend/app/routers/ontologies.py b/backend/app/routers/ontologies.py index 143d9b38..7d3fadf0 100644 --- a/backend/app/routers/ontologies.py +++ b/backend/app/routers/ontologies.py @@ -1,3 +1,5 @@ +import logging + from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy.orm import Session from sqlalchemy import func, text @@ -13,6 +15,7 @@ from app.services.publication.working_copy import OntologyWorkingCopyService import uuid +logger = logging.getLogger(__name__) router = APIRouter() @router.get("") @@ -82,9 +85,14 @@ def delete_ontology(ontology_id: str, db: Session = Depends(get_db), _=Depends(r # gone, unlike execution history (agent runs, sandbox simulations, # business-journey records, MCP write requests) rooted through # ontology_releases, which a delete must not silently wipe. + # entity_instance_relations (and entity_instances, cascade-deleted below) + # carry an AFTER DELETE trigger that re-emits agent_index_outbox rows — + # agent_index_outbox must be cleaned LAST so it sweeps up anything those + # triggers just recreated, or the delete below hits ONTOLOGY_HAS_DEPENDENT_DATA + # on an ontology that still has real instance data. for table in ( "ontology_data_grants", "ontology_project_access_grants", "ontology_migration_findings", - "agent_index_outbox", "entity_property_definitions", "entity_instance_relations", + "entity_property_definitions", "entity_instance_relations", "agent_index_outbox", ): db.execute(text(f"DELETE FROM {table} WHERE ontology_id = :id"), {"id": ontology_id}) try: @@ -93,3 +101,18 @@ def delete_ontology(ontology_id: str, db: Session = Depends(get_db), _=Depends(r except IntegrityError: db.rollback() raise HTTPException(409, detail="ONTOLOGY_HAS_DEPENDENT_DATA") + + # Best-effort external-store cleanup — never fails an already-committed + # delete; Neo4j/Chroma hold per-ontology graph/vector data with no other + # owner once the Postgres row is gone (mirrors the try/except-and-log + # pattern in app/tasks/extraction.py's _sync_neo4j). + try: + from app.services.v2.graph.neo4j_service import Neo4jService + Neo4jService().delete_by_ontology(ontology_id) + except Exception: + logger.warning("Neo4j cleanup failed for deleted ontology %s", ontology_id, exc_info=True) + try: + from app.services.v2.vector.chroma_service import ChromaService + ChromaService().delete_collection(f"ontology_{ontology_id}") + except Exception: + logger.warning("Chroma cleanup failed for deleted ontology %s", ontology_id, exc_info=True) diff --git a/backend/app/routers/v2/curated.py b/backend/app/routers/v2/curated.py index 53ee2cba..e0468041 100644 --- a/backend/app/routers/v2/curated.py +++ b/backend/app/routers/v2/curated.py @@ -79,7 +79,13 @@ def delete_curated(dataset_id: str, db: Session = Depends(get_db), _admin=Depend raise HTTPException(404, "Curated dataset not found") # 清理关联版本和媒体项 ver_ids = [v.id for v in db.query(DatasetVersion).filter(DatasetVersion.dataset_id == dataset_id).all()] + media_uris: list[str] = [] if ver_ids: + media_items = db.query(MediaItem).filter(MediaItem.dataset_version_id.in_(ver_ids)).all() + for m in media_items: + media_uris.append(m.storage_uri) + if m.ocr_result_uri: + media_uris.append(m.ocr_result_uri) db.query(MediaItem).filter(MediaItem.dataset_version_id.in_(ver_ids)).delete(synchronize_session=False) db.query(DatasetVersion).filter(DatasetVersion.dataset_id == dataset_id).delete(synchronize_session=False) # 清理审核记录 @@ -87,6 +93,16 @@ def delete_curated(dataset_id: str, db: Session = Depends(get_db), _admin=Depend db.delete(ds) db.commit() + # Best-effort MinIO cleanup — never fails an already-committed delete + if media_uris: + from app.services.storage_service import get_storage_service + storage = get_storage_service() + for uri in media_uris: + try: + storage.delete_object(uri) + except Exception: + logger.warning("MinIO cleanup failed for object %s (dataset %s)", uri, dataset_id, exc_info=True) + @router.get("/{dataset_id}", response_model=CuratedDatasetResponse) def get_curated(dataset_id: str, db: Session = Depends(get_db)): diff --git a/backend/tests/test_ontologies.py b/backend/tests/test_ontologies.py index 754a9506..4489dff6 100644 --- a/backend/tests/test_ontologies.py +++ b/backend/tests/test_ontologies.py @@ -1,3 +1,51 @@ +def test_delete_ontology_cleans_up_neo4j_and_chroma(client, auth_headers, monkeypatch): + """Deleting an ontology must also remove its Neo4j nodes/relations and its + Chroma vector collection — those stores have no other owner once the + Postgres row is gone, and previously nothing in the delete path ever + called their (already-implemented) delete methods.""" + calls = {} + + class FakeNeo4j: + def delete_by_ontology(self, ontology_id): + calls["neo4j_ontology_id"] = ontology_id + return 3 + + class FakeChroma: + def delete_collection(self, name): + calls["chroma_collection"] = name + return True + + monkeypatch.setattr("app.services.v2.graph.neo4j_service.Neo4jService", FakeNeo4j) + monkeypatch.setattr("app.services.v2.vector.chroma_service.ChromaService", FakeChroma) + + r = client.post("/api/v1/ontologies", json={"name": "CleanupTest", "domain": "财务"}, headers=auth_headers) + oid = r.json()["data"]["id"] + r2 = client.delete(f"/api/v1/ontologies/{oid}", headers=auth_headers) + assert r2.status_code == 204 + assert calls["neo4j_ontology_id"] == oid + assert calls["chroma_collection"] == f"ontology_{oid}" + + +def test_delete_ontology_survives_neo4j_and_chroma_failure(client, auth_headers, monkeypatch): + """External-store cleanup is best-effort: a Postgres delete that already + committed must still return 204 even if Neo4j/Chroma cleanup blows up.""" + class BrokenNeo4j: + def delete_by_ontology(self, ontology_id): + raise ConnectionError("neo4j unreachable") + + class BrokenChroma: + def delete_collection(self, name): + raise ConnectionError("chroma unreachable") + + monkeypatch.setattr("app.services.v2.graph.neo4j_service.Neo4jService", BrokenNeo4j) + monkeypatch.setattr("app.services.v2.vector.chroma_service.ChromaService", BrokenChroma) + + r = client.post("/api/v1/ontologies", json={"name": "CleanupFailTest", "domain": "财务"}, headers=auth_headers) + oid = r.json()["data"]["id"] + r2 = client.delete(f"/api/v1/ontologies/{oid}", headers=auth_headers) + assert r2.status_code == 204 + + def test_create_ontology(client, auth_headers): r = client.post("/api/v1/ontologies", json={"name": "供应链测试", "domain": "供应链"}, diff --git a/backend/tests/test_ontology_deletion_with_instances.py b/backend/tests/test_ontology_deletion_with_instances.py new file mode 100644 index 00000000..4dcfd3c0 --- /dev/null +++ b/backend/tests/test_ontology_deletion_with_instances.py @@ -0,0 +1,95 @@ +"""Deleting an ontology that has entity_instance_relations data must not hit +ONTOLOGY_HAS_DEPENDENT_DATA — the AFTER DELETE trigger on entity_instance_relations +(and entity_instances) re-emits agent_index_outbox rows, which must be swept up +by cleaning agent_index_outbox LAST, not before those tables. Needs a real +Postgres schema (SQLite has none of these triggers).""" +import os +import uuid +from urllib.parse import quote + +import pytest +from sqlalchemy import create_engine, text +from sqlalchemy.orm import sessionmaker + +from app.routers.ontologies import delete_ontology + +TEST_DATABASE_URL = os.environ.get("TEST_DATABASE_URL") +DEFAULT_DOMAIN = "00000000-0000-0000-0000-000000000001" + + +def _scoped_url(schema: str) -> str: + return f"{TEST_DATABASE_URL}?options={quote(f'-csearch_path={schema},public', safe='-=,')}" + + +@pytest.fixture +def schema(): + if not TEST_DATABASE_URL: + pytest.skip("TEST_DATABASE_URL required") + schema = "ontology_delete_" + uuid.uuid4().hex + engine = create_engine(TEST_DATABASE_URL) + with engine.begin() as connection: + connection.execute(text(f'CREATE SCHEMA "{schema}"')) + import subprocess + import sys + from pathlib import Path + backend_dir = Path(__file__).resolve().parents[1] + result = subprocess.run( + [sys.executable, "scripts/run_migrations.py", "upgrade", "head"], + cwd=backend_dir, env=dict(os.environ, DATABASE_URL=_scoped_url(schema)), + capture_output=True, text=True, + ) + assert result.returncode == 0, result.stderr + yield schema + with engine.begin() as connection: + connection.execute(text(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE')) + engine.dispose() + + +def test_delete_ontology_with_instance_relations_does_not_hit_dependent_data(schema, monkeypatch): + engine = create_engine(_scoped_url(schema)) + session = sessionmaker(bind=engine)() + + oid, u = "o-del-1", "u-del-1" + session.execute(text( + "INSERT INTO users (id,username,email,password_hash,role,is_active,security_domain_id,created_at,updated_at) " + "VALUES (:u,'d','d@t.com','h','admin',true,:d,now(),now())" + ), {"u": u, "d": DEFAULT_DOMAIN}) + session.execute(text( + "INSERT INTO ontology_projects (id,name,domain,version,status,created_by,created_at,updated_at,security_domain_id,working_revision) " + "VALUES (:o,'del-test','credit','v1','created',:u,now(),now(),:d,1)" + ), {"o": oid, "u": u, "d": DEFAULT_DOMAIN}) + session.execute(text( + "INSERT INTO entities (id, ontology_id, name_cn, type, description, properties, confidence, version, created_at, updated_at) " + "VALUES ('e-1', :o, '概念', 'Concept', 'd', '{}'::json, 0.9, 'v1', now(), now())" + ), {"o": oid}) + session.execute(text( + "INSERT INTO entity_instances (id, entity_id, ontology_id, row_identity, row_data, revision, created_at, updated_at) " + "VALUES (:i1, 'e-1', :o, 'inst-1', '{}'::json, 1, now(), now()), " + "(:i2, 'e-1', :o, 'inst-2', '{}'::json, 1, now(), now())" + ), {"o": oid, "i1": str(uuid.uuid4()), "i2": str(uuid.uuid4())}) + inst_ids = session.execute(text( + "SELECT id FROM entity_instances WHERE ontology_id = :o ORDER BY row_identity" + ), {"o": oid}).scalars().all() + session.execute(text( + "INSERT INTO relations (id, ontology_id, source_entity, target_entity, type, properties, confidence, created_at) " + "VALUES ('r-1', :o, 'e-1', 'e-1', 'related_to', '{}'::json, 0.9, now())" + ), {"o": oid}) + session.execute(text( + "INSERT INTO entity_instance_relations (id, ontology_id, source_instance_id, target_instance_id, relation_definition_id, properties, revision, created_at, updated_at) " + "VALUES (:id, :o, :s, :t, 'r-1', '{}'::json, 1, now(), now())" + ), {"id": str(uuid.uuid4()), "o": oid, "s": inst_ids[0], "t": inst_ids[1]}) + session.commit() + + # confirm the trigger fired on insert too (sanity: outbox has rows before delete) + outbox_before = session.execute(text( + "SELECT count(*) FROM agent_index_outbox WHERE ontology_id = :o" + ), {"o": oid}).scalar_one() + assert outbox_before > 0 + + delete_ontology(ontology_id=oid, db=session) + + remaining = session.execute(text( + "SELECT count(*) FROM ontology_projects WHERE id = :o" + ), {"o": oid}).scalar_one() + assert remaining == 0 + session.close() diff --git a/backend/tests/v2/test_curated_deletion.py b/backend/tests/v2/test_curated_deletion.py new file mode 100644 index 00000000..57d62f2f --- /dev/null +++ b/backend/tests/v2/test_curated_deletion.py @@ -0,0 +1,78 @@ +"""Curated dataset deletion — MinIO object cleanup""" +import uuid + +import pytest + +from app.models.v2.dataset import Dataset, DatasetVersion, MediaItem + + +@pytest.fixture +def curated_client(client, db): + """v2 curated 路由用模块内 get_db, 需单独覆盖到测试库""" + from app.main import app + from app.routers.v2 import curated + + def _override(): + yield db + + app.dependency_overrides[curated.get_db] = _override + yield client + app.dependency_overrides.pop(curated.get_db, None) + + +def _login(client, username, password): + r = client.post("/api/v1/auth/login", json={"username": username, "password": password}) + return {"Authorization": f"Bearer {r.json()['data']['access_token']}"} + + +def _make_curated_with_media(db) -> tuple[str, list[str]]: + ds = Dataset(id=str(uuid.uuid4()), name=f"del-{uuid.uuid4().hex[:6]}", kind="curated") + db.add(ds) + db.flush() + ver = DatasetVersion(id=str(uuid.uuid4()), dataset_id=ds.id, version_no=1) + db.add(ver) + db.flush() + uris = [f"s3://curated/{uuid.uuid4().hex}.pdf", f"s3://curated/{uuid.uuid4().hex}.docx"] + media = MediaItem( + id=str(uuid.uuid4()), dataset_version_id=ver.id, media_type="pdf", + storage_uri=uris[0], ocr_result_uri=uris[1], + ) + db.add(media) + db.commit() + return ds.id, uris + + +def test_delete_curated_cleans_up_minio_objects(curated_client, db, admin_user, monkeypatch): + """Deleting a curated dataset must remove its MediaItem MinIO objects + (storage_uri and ocr_result_uri) — previously nothing in the delete path + ever called StorageService.delete_object, leaving them as permanent + orphaned storage.""" + ds_id, expected_uris = _make_curated_with_media(db) + deleted = [] + + class FakeStorage: + def delete_object(self, uri): + deleted.append(uri) + + monkeypatch.setattr("app.services.storage_service.get_storage_service", lambda: FakeStorage()) + + headers = _login(curated_client, "admin", "admin123") + r = curated_client.delete(f"/api/v2/curated/{ds_id}", headers=headers) + assert r.status_code == 204 + assert sorted(deleted) == sorted(expected_uris) + + +def test_delete_curated_survives_minio_failure(curated_client, db, admin_user, monkeypatch): + """MinIO cleanup is best-effort: a Postgres delete that already committed + must still return 204 even if object storage cleanup blows up.""" + ds_id, _ = _make_curated_with_media(db) + + class BrokenStorage: + def delete_object(self, uri): + raise ConnectionError("minio unreachable") + + monkeypatch.setattr("app.services.storage_service.get_storage_service", lambda: BrokenStorage()) + + headers = _login(curated_client, "admin", "admin123") + r = curated_client.delete(f"/api/v2/curated/{ds_id}", headers=headers) + assert r.status_code == 204 From 8addf592e271966368b13cb1cd106031712e2d0a Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Thu, 17 Sep 2026 15:49:36 +0800 Subject: [PATCH 14/27] fix(pipeline-mapping): stop treating any 200 response as a passed connection test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported by a user: testing a Pipeline Mapping data-source connection always showed "success" regardless of host/port/username/password. The backend (app/routers/v2/connections.py's /connections/test-config) was never the problem — it genuinely attempts the connection and returns HTTP 200 with {"success": false, "detail": "..."} on failure rather than raising, so wrong credentials or an unreachable host still come back as a normal 2xx response. The frontend's test-connection button awaited that response and, since axios only rejects on non-2xx, treated any resolved promise as success without ever looking at the `success` field in the body — so the only way to see "failed" was a network error or an auth/validation rejection, never an actual bad connection. Now the button reads `result.success` and falls back to the failed state (using the backend's `detail` message) when it's false. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01AkAraQUPvCkrAKpk89TMNR --- .../inspectors/ConnectorInspector.test.tsx | 35 +++++++++++++++++++ .../builder/inspectors/ConnectorInspector.tsx | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 frontend/src/pages/pipelines/builder/inspectors/ConnectorInspector.test.tsx diff --git a/frontend/src/pages/pipelines/builder/inspectors/ConnectorInspector.test.tsx b/frontend/src/pages/pipelines/builder/inspectors/ConnectorInspector.test.tsx new file mode 100644 index 00000000..af214f6a --- /dev/null +++ b/frontend/src/pages/pipelines/builder/inspectors/ConnectorInspector.test.tsx @@ -0,0 +1,35 @@ +import '@/i18n' +import { describe, expect, it, vi } from 'vitest' +import { render, screen, fireEvent, waitFor } from '@testing-library/react' +import { apiClientV2 } from '@/api/client' +import ConnectorInspector from './ConnectorInspector' + +vi.mock('@/api/client', () => ({ + apiClientV2: { post: vi.fn() }, +})) + +describe('ConnectorInspector test-connection button', () => { + const dbConfig = { source_type: 'postgresql', config_values: { host: 'localhost', port: '5432' } } + + it('shows failed when the backend reports success: false (e.g. wrong credentials) instead of treating any 200 response as success', async () => { + vi.mocked(apiClientV2.post).mockResolvedValueOnce({ success: false, detail: 'connection refused' }) + render( {}} />) + + fireEvent.click(screen.getByRole('button', { name: /测试连接|Test Connection/i })) + + await waitFor(() => { + expect(screen.getByText('connection refused')).toBeTruthy() + }) + }) + + it('shows success only when the backend actually reports success: true', async () => { + vi.mocked(apiClientV2.post).mockResolvedValueOnce({ success: true }) + render( {}} />) + + fireEvent.click(screen.getByRole('button', { name: /测试连接|Test Connection/i })) + + await waitFor(() => { + expect(screen.getByRole('button', { name: /连接成功|success/i })).toBeTruthy() + }) + }) +}) diff --git a/frontend/src/pages/pipelines/builder/inspectors/ConnectorInspector.tsx b/frontend/src/pages/pipelines/builder/inspectors/ConnectorInspector.tsx index 66f5cb37..b6d86c97 100644 --- a/frontend/src/pages/pipelines/builder/inspectors/ConnectorInspector.tsx +++ b/frontend/src/pages/pipelines/builder/inspectors/ConnectorInspector.tsx @@ -102,7 +102,7 @@ export default function ConnectorInspector({ config, onChange, readOnly = false )} {sourceType !== 'file' && (
{DB_CONFIG_FIELDS[sourceType]?.map(f => (
| undefined)?.[f.key] || '')} onChange={e => { const cv2 = { ...((config.config_values as Record | undefined) || {}), [f.key]: e.target.value }; onChange('config_values', cv2); setTestStatus('idle') }} placeholder={f.placeholder} className="w-full border rounded-lg px-3 py-1.5 text-sm" />
))}
)}
-
From 28f2a9cab27459218d70d3157894e1facd859b38 Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Thu, 17 Sep 2026 19:29:12 +0800 Subject: [PATCH 15/27] fix(curated): delete the legacy v2_curated_datasets mirror row too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported gap (#5 from earlier community feedback triage): deleting a curated dataset left behind a permanent orphan once it had ever gone through review. _ensure_curated_dataset_row (and the older /review endpoint's inline equivalent) lazily creates a v2_curated_datasets row sharing the SAME id as the real v2_datasets row, purely as an FK bridge so CuratedReview has something to point at — nothing else links the two tables. delete_curated only ever deleted the v2_datasets row and the CuratedReview rows filtered by that shared id; the CuratedDataset mirror row itself was never targeted by any delete path, so a reviewed-then-deleted dataset always left one behind. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01AkAraQUPvCkrAKpk89TMNR --- backend/app/routers/v2/curated.py | 4 +++ backend/tests/v2/test_curated_deletion.py | 30 ++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/backend/app/routers/v2/curated.py b/backend/app/routers/v2/curated.py index e0468041..2eabe4fe 100644 --- a/backend/app/routers/v2/curated.py +++ b/backend/app/routers/v2/curated.py @@ -90,6 +90,10 @@ def delete_curated(dataset_id: str, db: Session = Depends(get_db), _admin=Depend db.query(DatasetVersion).filter(DatasetVersion.dataset_id == dataset_id).delete(synchronize_session=False) # 清理审核记录 db.query(CuratedReview).filter(CuratedReview.curated_dataset_id == dataset_id).delete(synchronize_session=False) + # 若曾发起过审核,_ensure_curated_dataset_row/旧版 /review 端点会用相同 id + # 懒创建一条 v2_curated_datasets 镜像行作为 CuratedReview 的 FK 桥接——上面 + # 只删了 v2_datasets 里的真本体行,这条镜像行没有别的清理入口,不删就永久残留 + db.query(CuratedDataset).filter(CuratedDataset.id == dataset_id).delete(synchronize_session=False) db.delete(ds) db.commit() diff --git a/backend/tests/v2/test_curated_deletion.py b/backend/tests/v2/test_curated_deletion.py index 57d62f2f..355fbe1a 100644 --- a/backend/tests/v2/test_curated_deletion.py +++ b/backend/tests/v2/test_curated_deletion.py @@ -1,8 +1,10 @@ -"""Curated dataset deletion — MinIO object cleanup""" +"""Curated dataset deletion — MinIO object cleanup and legacy-table cleanup""" import uuid +from datetime import datetime, timezone import pytest +from app.models.v2.curated import CuratedDataset, CuratedReview from app.models.v2.dataset import Dataset, DatasetVersion, MediaItem @@ -62,6 +64,32 @@ def delete_object(self, uri): assert sorted(deleted) == sorted(expected_uris) +def test_delete_curated_removes_legacy_mirror_row(curated_client, db, admin_user, monkeypatch): + """Once a dataset has been through review, _ensure_curated_dataset_row + (and the older /review endpoint) lazily creates a v2_curated_datasets + row with the SAME id as the real v2_datasets row, purely as an FK bridge + for CuratedReview. Deleting the dataset must remove that mirror row too + — previously delete_curated only ever deleted the v2_datasets row and + the CuratedReview rows, leaving the CuratedDataset mirror as a permanent + orphan.""" + monkeypatch.setattr("app.services.storage_service.get_storage_service", lambda: object()) + ds_id, _ = _make_curated_with_media(db) + # simulate what a completed review leaves behind + db.add(CuratedDataset(id=ds_id, name="mirror", status="approved")) + db.add(CuratedReview( + id=str(uuid.uuid4()), curated_dataset_id=ds_id, status="approved", + decided_at=datetime.now(timezone.utc), + )) + db.commit() + + headers = _login(curated_client, "admin", "admin123") + r = curated_client.delete(f"/api/v2/curated/{ds_id}", headers=headers) + assert r.status_code == 204 + + assert db.query(CuratedDataset).filter(CuratedDataset.id == ds_id).first() is None + assert db.query(CuratedReview).filter(CuratedReview.curated_dataset_id == ds_id).first() is None + + def test_delete_curated_survives_minio_failure(curated_client, db, admin_user, monkeypatch): """MinIO cleanup is best-effort: a Postgres delete that already committed must still return 204 even if object storage cleanup blows up.""" From c2831aa4ece59731410aba78eca12575c9483219 Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Thu, 17 Sep 2026 21:33:32 +0800 Subject: [PATCH 16/27] fix(ontology): stop the delete-failure messages from suggesting impossible fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported gap (#3 from earlier community feedback triage): the two ontology-delete failure messages told users to do something that no mechanism in the app can actually do. ONTOLOGY_BOUND_TO_AGENT said "unbind the agent first" — no unbind capability exists anywhere (backend or frontend); the only code that ever deletes an agent_ontology_bindings row is the agent's full hard-delete. Building a real unbind would mean letting a historical agent version's ontology binding be removed, which conflicts with this codebase's explicit agent-version immutability guarantee ("old versions stay byte-identical") — the same category of decision as the release-immutability question (#2), deliberately deferred rather than solved here. ONTOLOGY_HAS_DEPENDENT_DATA said "clean up first" — the dependent data is exactly the execution history rooted through ontology_releases, which a DB trigger (reject_ontology_release_mutation) makes unconditionally immutable with zero exception path. Both messages now state the actual situation instead of pointing at a remedy that doesn't exist, in both zh and en. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01AkAraQUPvCkrAKpk89TMNR --- frontend/src/i18n/en.json | 4 ++-- frontend/src/i18n/zh.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/i18n/en.json b/frontend/src/i18n/en.json index 686d39c7..059c55f6 100644 --- a/frontend/src/i18n/en.json +++ b/frontend/src/i18n/en.json @@ -124,8 +124,8 @@ "confirm_delete_msg": "Delete ontology \"{{name}}\"? This cannot be undone.", "cancel": "Cancel", "confirm": "Delete", - "delete_failed_bound_agent": "Cannot delete: this ontology is bound to an Agent — unbind it first", - "delete_failed_dependent_data": "Cannot delete: this ontology has dependent execution records (runs, sandbox simulations, etc.) — clean those up first", + "delete_failed_bound_agent": "Cannot delete: this ontology is bound to an Agent version, and Agent version bindings can never be changed once created — so this ontology cannot be deleted", + "delete_failed_dependent_data": "Cannot delete: this ontology has published releases with dependent execution records (runs, sandbox simulations, etc.) that are kept permanently for audit purposes — so this ontology cannot be deleted", "delete_failed_generic": "Delete failed, please try again", "deleting": "Deleting…", "select_all": "Select all", diff --git a/frontend/src/i18n/zh.json b/frontend/src/i18n/zh.json index c5a393ed..d9b50b05 100644 --- a/frontend/src/i18n/zh.json +++ b/frontend/src/i18n/zh.json @@ -124,8 +124,8 @@ "confirm_delete_msg": "确定要删除本体「{{name}}」吗?此操作不可撤销。", "cancel": "取消", "confirm": "确认删除", - "delete_failed_bound_agent": "无法删除:该本体已绑定至智能体,请先解绑对应智能体后再删除", - "delete_failed_dependent_data": "无法删除:该本体存在关联的执行记录(如运行历史、沙盒模拟等),请先清理后再删除", + "delete_failed_bound_agent": "无法删除:该本体已被某个智能体版本绑定,智能体版本一经创建绑定即不可更改,因此这个本体无法删除", + "delete_failed_dependent_data": "无法删除:该本体存在已发布版本及关联的执行记录(运行历史、沙盒模拟等),出于审计要求这些记录会永久保留,因此本体无法删除", "delete_failed_generic": "删除失败,请稍后重试", "deleting": "删除中…", "select_all": "全选", From 3b5be6ebf1a229a3b29f9993b8dea30e0bd67649 Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Fri, 13 Nov 2026 08:29:22 +0800 Subject: [PATCH 17/27] fix(security): enforce per-ontology project grants on legacy and v2 routes Legacy v1 ontology routers (entities, logic, actions, graph, files, export, audit, extraction, ontology get/update/delete) and the classic v2 graph, search and logic/action routers only required a logged-in user. Any account, including a self-registered viewer, could list ontology ids via /overview/stats and read or export every ontology; editors could modify ontologies they had no grant on. Every route now checks the caller's ontology_project_access_grants row (read or edit) and answers 404 ONTOLOGY_NOT_FOUND when it is missing, so the existence of other ontologies is not disclosed. Overview stats only count discoverable ontologies, and Neo4j traversals stay within the ontology they were asked about. Ontologies inserted by direct SQL have no creator grant; run scripts/backfill_creator_grants.py after deploying. Co-Authored-By: Claude Opus 5.5 --- backend/app/routers/actions.py | 21 ++- backend/app/routers/audit.py | 14 +- backend/app/routers/entities.py | 20 +- backend/app/routers/export.py | 7 +- backend/app/routers/extraction.py | 13 +- backend/app/routers/files.py | 19 +- backend/app/routers/graph.py | 30 ++- backend/app/routers/logic.py | 21 ++- backend/app/routers/ontologies.py | 36 +++- backend/app/routers/overview.py | 24 ++- backend/app/routers/v2/graph.py | 39 ++-- backend/app/routers/v2/logic_actions.py | 53 ++++-- backend/app/routers/v2/search.py | 17 +- backend/app/services/ontology_access.py | 115 +++++++++++- .../app/services/v2/graph/graph_analytics.py | 12 +- .../agent/test_core_route_registration.py | 3 + .../agent/test_import_mutation_closure.py | 5 + .../tests/agent/test_legacy_route_closure.py | 3 + .../tests/agent/test_legacy_runtime_tables.py | 3 + .../tests/agent/test_ontology_access_admin.py | 172 +++++++++++++++++- .../tests/agent/test_ontology_route_access.py | 80 ++++++++ .../test_ontology_deletion_with_instances.py | 16 +- backend/tests/v2/graph/test_analytics.py | 36 ++++ .../tests/v2/test_logic_actions_runtime.py | 12 +- 24 files changed, 679 insertions(+), 92 deletions(-) create mode 100644 backend/tests/agent/test_ontology_route_access.py diff --git a/backend/app/routers/actions.py b/backend/app/routers/actions.py index a923fc67..cf3bb8fa 100644 --- a/backend/app/routers/actions.py +++ b/backend/app/routers/actions.py @@ -2,6 +2,7 @@ from sqlalchemy.orm import Session from app.deps import get_db, get_current_user, require_editor from app.models.user import User +from app.services.ontology_access import GrantDenied, require_project_grant from app.services.publication.working_copy import OntologyWorkingCopyService from app.services.publication.legacy_sync import delete_action_mirror, sync_action_to_v2 from app.models.action import Action @@ -10,13 +11,22 @@ router = APIRouter() + +def _grant(db: Session, actor: User, ontology_id: str, capability: str) -> dict: + try: + return require_project_grant(db, actor, ontology_id, capability) + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + @router.get("") -def list_actions(ontology_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def list_actions(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") items = db.query(Action).filter(Action.ontology_id == ontology_id).all() return {"data": [ActionOut.model_validate(a).model_dump() for a in items]} @router.post("", status_code=201) def create_action(ontology_id: str, body: ActionCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") def _write(): data = {k: v for k, v in body.model_dump().items() if v is not None} a = Action(id=str(uuid.uuid4()), ontology_id=ontology_id, **data) @@ -29,7 +39,8 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="action.create", callback=_write) @router.get("/{action_id}") -def get_action(ontology_id: str, action_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def get_action(ontology_id: str, action_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") a = db.query(Action).filter(Action.id == action_id, Action.ontology_id == ontology_id).first() if not a: raise HTTPException(404, "Not found") @@ -37,6 +48,7 @@ def get_action(ontology_id: str, action_id: str, db: Session = Depends(get_db), @router.put("/{action_id}") def update_action(ontology_id: str, action_id: str, body: ActionUpdate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") a = db.query(Action).filter(Action.id == action_id, Action.ontology_id == ontology_id).first() if not a: raise HTTPException(404, "Not found") @@ -51,6 +63,7 @@ def _write(): @router.delete("/{action_id}", status_code=204) def delete_action(ontology_id: str, action_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") a = db.query(Action).filter(Action.id == action_id, Action.ontology_id == ontology_id).first() if not a: raise HTTPException(404, "Not found") @@ -62,6 +75,7 @@ def _write(): @router.post("/{action_id}/toggle") def toggle_action(ontology_id: str, action_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") a = db.query(Action).filter(Action.id == action_id, Action.ontology_id == ontology_id).first() if not a: raise HTTPException(404, "Not found") @@ -74,6 +88,7 @@ def _write(): @router.post("/publish") -def publish_actions(ontology_id: str, db: Session = Depends(get_db), _=Depends(require_editor)): +def publish_actions(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): """Direct tool publication is closed; use the Ontology compiler (P1C).""" + _grant(db, current_user, ontology_id, "edit") raise HTTPException(status_code=403, detail="PUBLICATION_NOT_ENABLED") diff --git a/backend/app/routers/audit.py b/backend/app/routers/audit.py index 88246768..5ee5c3ac 100644 --- a/backend/app/routers/audit.py +++ b/backend/app/routers/audit.py @@ -5,13 +5,19 @@ from app.models.ontology import OntologyProject from app.schemas.audit import AuditRequest, AuditTaskOut from app.services.publication.extraction_model_port import default_extraction_model_port +from app.models.user import User +from app.services.ontology_access import GrantDenied, require_project_grant import uuid router = APIRouter() @router.post("") -def start_audit(ontology_id: str, body: AuditRequest, db: Session = Depends(get_db), _=Depends(require_editor)): +def start_audit(ontology_id: str, body: AuditRequest, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + try: + require_project_grant(db, current_user, ontology_id, "edit") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc project = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first() if not project: raise HTTPException(404, "Ontology not found") @@ -50,7 +56,11 @@ def run_sync(): @router.get("/status") -def get_audit_status(ontology_id: str, task_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def get_audit_status(ontology_id: str, task_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + try: + require_project_grant(db, current_user, ontology_id, "read") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc task = db.query(AuditTask).filter(AuditTask.id == task_id, AuditTask.ontology_id == ontology_id).first() if not task: raise HTTPException(404, "Audit task not found") diff --git a/backend/app/routers/entities.py b/backend/app/routers/entities.py index d8651313..d3fa0358 100644 --- a/backend/app/routers/entities.py +++ b/backend/app/routers/entities.py @@ -3,19 +3,29 @@ from app.deps import get_db, get_current_user, require_admin, require_editor from app.models.entity import Entity from app.models.user import User +from app.services.ontology_access import GrantDenied, require_project_grant from app.services.publication.working_copy import OntologyWorkingCopyService from app.schemas.entity import EntityCreate, EntityUpdate, EntityOut import uuid router = APIRouter() + +def _grant(db: Session, actor: User, ontology_id: str, capability: str) -> dict: + try: + return require_project_grant(db, actor, ontology_id, capability) + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + @router.get("") -def list_entities(ontology_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def list_entities(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") items = db.query(Entity).filter(Entity.ontology_id == ontology_id).all() return {"data": [EntityOut.model_validate(e).model_dump() for e in items]} @router.post("", status_code=201) def create_entity(ontology_id: str, body: EntityCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") def _write(): data = {k: v for k, v in body.model_dump().items() if v is not None} e = Entity(id=str(uuid.uuid4()), ontology_id=ontology_id, **data) @@ -24,7 +34,8 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="entity.create", callback=_write) @router.get("/{entity_id}") -def get_entity(ontology_id: str, entity_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def get_entity(ontology_id: str, entity_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") e = db.query(Entity).filter(Entity.id == entity_id, Entity.ontology_id == ontology_id).first() if not e: raise HTTPException(404, "Not found") @@ -32,6 +43,7 @@ def get_entity(ontology_id: str, entity_id: str, db: Session = Depends(get_db), @router.put("/{entity_id}") def update_entity(ontology_id: str, entity_id: str, body: EntityUpdate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") e = db.query(Entity).filter(Entity.id == entity_id, Entity.ontology_id == ontology_id).first() if not e: raise HTTPException(404, "Not found") @@ -44,6 +56,7 @@ def _write(): @router.delete("/{entity_id}", status_code=204) def delete_entity(ontology_id: str, entity_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_admin)): + _grant(db, current_user, ontology_id, "edit") e = db.query(Entity).filter(Entity.id == entity_id, Entity.ontology_id == ontology_id).first() if not e: raise HTTPException(404, "Not found") @@ -56,9 +69,10 @@ def list_entity_instances( ontology_id: str, entity_id: str, db: Session = Depends(get_db), - _=Depends(get_current_user), + current_user: User = Depends(get_current_user), ): """概念实体下挂的行级实例数据(Pipeline Mapping 与简易 LLM 提取共用同一张表)""" + _grant(db, current_user, ontology_id, "read") from app.models.entity_instance import EntityInstance entity = db.query(Entity).filter( diff --git a/backend/app/routers/export.py b/backend/app/routers/export.py index 8604fdc2..ae6078e0 100644 --- a/backend/app/routers/export.py +++ b/backend/app/routers/export.py @@ -3,6 +3,7 @@ from sqlalchemy.orm import Session from app.deps import get_db, get_current_user from app.services import export_service +from app.services.ontology_access import GrantDenied, require_project_grant router = APIRouter() @@ -21,8 +22,12 @@ def export_ontology( ontology_id: str, format: str = Query("json", pattern="^(json|yaml|csv|ttl|html|cypher|tugraph)$"), db: Session = Depends(get_db), - _=Depends(get_current_user) + current_user=Depends(get_current_user) ): + try: + require_project_grant(db, current_user, ontology_id, "read") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc if format not in FORMAT_MAP: raise HTTPException(400, f"Unsupported format: {format}") content_type, ext, fn = FORMAT_MAP[format] diff --git a/backend/app/routers/extraction.py b/backend/app/routers/extraction.py index 81a27ec2..6d9828d6 100644 --- a/backend/app/routers/extraction.py +++ b/backend/app/routers/extraction.py @@ -9,6 +9,7 @@ from app.models.ontology import OntologyProject from app.services.document_service import combine_converted_files from app.schemas.extraction import ExtractionRequest, ExtractionTaskOut +from app.services.ontology_access import GrantDenied, require_project_grant import uuid logger = logging.getLogger(__name__) @@ -16,7 +17,11 @@ router = APIRouter() @router.post("") -def start_extraction(ontology_id: str, body: ExtractionRequest, db: Session = Depends(get_db), _=Depends(require_editor)): +def start_extraction(ontology_id: str, body: ExtractionRequest, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + try: + require_project_grant(db, current_user, ontology_id, "edit") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc project = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first() if not project: raise HTTPException(404, "Ontology not found") @@ -67,7 +72,11 @@ def run_sync(): return {"data": {"task_id": task.id}, "message": "Extraction queued"} @router.get("/status") -def get_extraction_status(ontology_id: str, task_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def get_extraction_status(ontology_id: str, task_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + try: + require_project_grant(db, current_user, ontology_id, "read") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc task = db.query(ExtractionTask).filter(ExtractionTask.id == task_id, ExtractionTask.ontology_id == ontology_id).first() if not task: raise HTTPException(404, "Task not found") diff --git a/backend/app/routers/files.py b/backend/app/routers/files.py index 926e1d14..4c79d6a9 100644 --- a/backend/app/routers/files.py +++ b/backend/app/routers/files.py @@ -7,6 +7,7 @@ from app.models.ontology import OntologyProject from app.schemas.file import FileOut from app.services.document_service import convert_document, is_usable_converted_text +from app.services.ontology_access import GrantDenied, require_project_grant from app.config import settings router = APIRouter() @@ -39,8 +40,16 @@ def _file_out(f: UploadedFile) -> dict: ).model_dump() +def _grant(db: Session, actor, ontology_id: str, capability: str) -> dict: + try: + return require_project_grant(db, actor, ontology_id, capability) + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + + @router.get("") -def list_files(ontology_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def list_files(ontology_id: str, db: Session = Depends(get_db), current_user=Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") files = db.query(UploadedFile).filter(UploadedFile.ontology_id == ontology_id).all() return {"data": [_file_out(f) for f in files]} @@ -49,11 +58,12 @@ async def upload_file( ontology_id: str, file: UploadFile = File(...), db: Session = Depends(get_db), - _=Depends(require_editor) + current_user=Depends(require_editor), ): + _grant(db, current_user, ontology_id, "edit") project = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first() if not project: - raise HTTPException(404, "Ontology not found") + raise HTTPException(404, detail="ONTOLOGY_NOT_FOUND") ext_name = (file.filename or "").rsplit(".", 1)[-1].lower() allowed = {e.strip() for e in settings.allowed_upload_extensions.split(",") if e.strip()} @@ -93,7 +103,8 @@ async def upload_file( return {"data": _file_out(db_file)} @router.delete("/{file_id}", status_code=204) -def delete_file(ontology_id: str, file_id: str, db: Session = Depends(get_db), _=Depends(require_admin)): +def delete_file(ontology_id: str, file_id: str, db: Session = Depends(get_db), current_user=Depends(require_admin)): + _grant(db, current_user, ontology_id, "edit") f = db.query(UploadedFile).filter(UploadedFile.id == file_id, UploadedFile.ontology_id == ontology_id).first() if not f: raise HTTPException(404, "File not found") diff --git a/backend/app/routers/graph.py b/backend/app/routers/graph.py index ea6bca00..61aeee57 100644 --- a/backend/app/routers/graph.py +++ b/backend/app/routers/graph.py @@ -5,13 +5,22 @@ from app.models.relation import Relation from app.models.ontology import OntologyProject from app.models.user import User +from app.services.ontology_access import GrantDenied, require_project_grant from app.services.publication.working_copy import OntologyWorkingCopyService from app.services.relation_dedup import dedupe_relation_rows router = APIRouter() + +def _grant(db: Session, actor: User, ontology_id: str, capability: str) -> dict: + try: + return require_project_grant(db, actor, ontology_id, capability) + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + @router.get("") -def get_graph(ontology_id: str, limit: int = 300, db: Session = Depends(get_db), _=Depends(get_current_user)): +def get_graph(ontology_id: str, limit: int = 300, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") project = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first() if not project: raise HTTPException(404, "Ontology not found") @@ -138,12 +147,26 @@ def create_relation( ): from app.models.relation import Relation import uuid + _grant(db, current_user, ontology_id, "edit") + source_entity = body.get("source_entity") + target_entity = body.get("target_entity") + if not source_entity or not target_entity: + raise HTTPException(status_code=422, detail="source_entity and target_entity are required") + source = db.query(Entity).filter( + Entity.id == source_entity, Entity.ontology_id == ontology_id + ).first() + target = db.query(Entity).filter( + Entity.id == target_entity, Entity.ontology_id == ontology_id + ).first() + if source is None or target is None: + raise HTTPException(status_code=422, detail="RELATION_ENTITIES_MUST_BELONG_TO_ONTOLOGY") + def _write(): relation = Relation( id=str(uuid.uuid4()), ontology_id=ontology_id, - source_entity=body["source_entity"], - target_entity=body["target_entity"], + source_entity=source_entity, + target_entity=target_entity, type=body.get("type", "关联"), properties=body.get("properties", {}), confidence=body.get("confidence", 1.0), @@ -154,6 +177,7 @@ def _write(): @router.delete("/relations/{relation_id}", status_code=204) def delete_relation(ontology_id: str, relation_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_admin)): + _grant(db, current_user, ontology_id, "edit") r = db.query(Relation).filter(Relation.id == relation_id, Relation.ontology_id == ontology_id).first() if not r: raise HTTPException(404, "Not found") diff --git a/backend/app/routers/logic.py b/backend/app/routers/logic.py index f2e9b51b..8b3e867b 100644 --- a/backend/app/routers/logic.py +++ b/backend/app/routers/logic.py @@ -2,6 +2,7 @@ from sqlalchemy.orm import Session from app.deps import get_db, get_current_user, require_editor from app.models.user import User +from app.services.ontology_access import GrantDenied, require_project_grant from app.services.publication.working_copy import OntologyWorkingCopyService from app.services.publication.legacy_sync import delete_logic_rule_mirror, sync_logic_rule_to_v2 from app.models.logic import LogicRule @@ -10,13 +11,22 @@ router = APIRouter() + +def _grant(db: Session, actor: User, ontology_id: str, capability: str) -> dict: + try: + return require_project_grant(db, actor, ontology_id, capability) + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + @router.get("") -def list_logic(ontology_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def list_logic(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") items = db.query(LogicRule).filter(LogicRule.ontology_id == ontology_id).all() return {"data": [LogicRuleOut.model_validate(r).model_dump() for r in items]} @router.post("", status_code=201) def create_logic(ontology_id: str, body: LogicRuleCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") def _write(): data = {k: v for k, v in body.model_dump().items() if v is not None} r = LogicRule(id=str(uuid.uuid4()), ontology_id=ontology_id, **data) @@ -29,7 +39,8 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="logic.create", callback=_write) @router.get("/{logic_id}") -def get_logic(ontology_id: str, logic_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def get_logic(ontology_id: str, logic_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") r = db.query(LogicRule).filter(LogicRule.id == logic_id, LogicRule.ontology_id == ontology_id).first() if not r: raise HTTPException(404, "Not found") @@ -37,6 +48,7 @@ def get_logic(ontology_id: str, logic_id: str, db: Session = Depends(get_db), _= @router.put("/{logic_id}") def update_logic(ontology_id: str, logic_id: str, body: LogicRuleUpdate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") r = db.query(LogicRule).filter(LogicRule.id == logic_id, LogicRule.ontology_id == ontology_id).first() if not r: raise HTTPException(404, "Not found") @@ -51,6 +63,7 @@ def _write(): @router.delete("/{logic_id}", status_code=204) def delete_logic(ontology_id: str, logic_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") r = db.query(LogicRule).filter(LogicRule.id == logic_id, LogicRule.ontology_id == ontology_id).first() if not r: raise HTTPException(404, "Not found") @@ -63,6 +76,7 @@ def _write(): @router.post("/{logic_id}/toggle") def toggle_logic(ontology_id: str, logic_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): """Human Review: 启用/禁用规则""" + _grant(db, current_user, ontology_id, "edit") r = db.query(LogicRule).filter(LogicRule.id == logic_id, LogicRule.ontology_id == ontology_id).first() if not r: raise HTTPException(404, "Not found") @@ -75,6 +89,7 @@ def _write(): @router.post("/publish") -def publish_logic_rules(ontology_id: str, db: Session = Depends(get_db), _=Depends(require_editor)): +def publish_logic_rules(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): """Human Review: 发布所有草稿规则 — direct tool publication is closed; use the Ontology compiler (P1C).""" + _grant(db, current_user, ontology_id, "edit") raise HTTPException(status_code=403, detail="PUBLICATION_NOT_ENABLED") diff --git a/backend/app/routers/ontologies.py b/backend/app/routers/ontologies.py index 7d3fadf0..62ef6360 100644 --- a/backend/app/routers/ontologies.py +++ b/backend/app/routers/ontologies.py @@ -11,20 +11,34 @@ from app.models.relation import Relation from app.models.user import User from app.schemas.ontology import OntologyCreate, OntologyOut, OntologyListItem, OntologyUpdate -from app.services.ontology_access import creator_grant +from app.services.ontology_access import ( + GrantDenied, + creator_grant, + discoverable_ontology_ids, + require_project_grant, +) from app.services.publication.working_copy import OntologyWorkingCopyService import uuid logger = logging.getLogger(__name__) router = APIRouter() + +def _grant(db: Session, actor: User, ontology_id: str, capability: str) -> dict: + try: + return require_project_grant(db, actor, ontology_id, capability) + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + @router.get("") def list_ontologies( name: Optional[str] = None, page: int = 1, page_size: int = 20, - db: Session = Depends(get_db), _=Depends(get_current_user) + db: Session = Depends(get_db), current_user: User = Depends(get_current_user) ): - q = db.query(OntologyProject) + q = db.query(OntologyProject).filter( + OntologyProject.id.in_(discoverable_ontology_ids(db, current_user)) + ) if name: q = q.filter(OntologyProject.name.ilike(f"%{name}%")) total = q.count() @@ -44,7 +58,8 @@ def create_ontology(body: OntologyCreate, db: Session = Depends(get_db), current raise HTTPException(status_code=409, detail={"error": "DUPLICATE_NAME", "message": f"Ontology 名称「{body.name}」已存在", "existing_id": existing.id}) project = OntologyProject(id=str(uuid.uuid4()), name=body.name, domain=body.domain, description=body.description, build_mode=body.build_mode or "simple_llm", - created_by=current_user.id) + created_by=current_user.id, + security_domain_id=current_user.security_domain_id) db.add(project); db.flush() # atomically insert the exact creator grant (P1A-ACCESS contract) creator_grant(db, project.id, current_user) @@ -52,17 +67,19 @@ def create_ontology(body: OntologyCreate, db: Session = Depends(get_db), current return {"data": OntologyOut.model_validate(project).model_dump()} @router.get("/{ontology_id}") -def get_ontology(ontology_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def get_ontology(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") p = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first() if not p: - raise HTTPException(404, "Not found") + raise HTTPException(404, detail="ONTOLOGY_NOT_FOUND") return {"data": OntologyOut.model_validate(p).model_dump()} @router.put("/{ontology_id}") def update_ontology(ontology_id: str, body: OntologyUpdate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") p = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first() if not p: - raise HTTPException(404, "Not found") + raise HTTPException(404, detail="ONTOLOGY_NOT_FOUND") def _write(): for k, v in body.model_dump(exclude_none=True).items(): setattr(p, k, v) @@ -71,10 +88,11 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="ontology.update", callback=_write) @router.delete("/{ontology_id}", status_code=204) -def delete_ontology(ontology_id: str, db: Session = Depends(get_db), _=Depends(require_admin)): +def delete_ontology(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_admin)): + _grant(db, current_user, ontology_id, "edit") p = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first() if not p: - raise HTTPException(404, "Not found") + raise HTTPException(404, detail="ONTOLOGY_NOT_FOUND") bound_agent = db.execute(text( "SELECT 1 FROM agent_ontology_bindings WHERE ontology_id = :id LIMIT 1" ), {"id": ontology_id}).scalar_one_or_none() diff --git a/backend/app/routers/overview.py b/backend/app/routers/overview.py index 18247dc6..420f4dba 100644 --- a/backend/app/routers/overview.py +++ b/backend/app/routers/overview.py @@ -3,23 +3,29 @@ from sqlalchemy import func from app.deps import get_db, get_current_user from app.models.ontology import OntologyProject +from app.models.user import User +from app.services.ontology_access import discoverable_ontology_ids router = APIRouter() -def _safe_count(db, model): +def _safe_count(db, model, ontology_ids): try: - return db.query(model).count() + return db.query(model).filter(model.ontology_id.in_(ontology_ids)).count() except Exception: return 0 @router.get("/stats") -def get_stats(db: Session = Depends(get_db), _=Depends(get_current_user)): +def get_stats(db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): from app.models.entity import Entity from app.models.logic import LogicRule from app.models.action import Action + discoverable_ids = discoverable_ontology_ids(db, current_user) + # Recent ontologies - recent = db.query(OntologyProject).order_by(OntologyProject.updated_at.desc()).limit(6).all() + recent = db.query(OntologyProject).filter( + OntologyProject.id.in_(discoverable_ids) + ).order_by(OntologyProject.updated_at.desc()).limit(6).all() recent_list = [] for o in recent: entity_count = db.query(Entity).filter(Entity.ontology_id == o.id).count() @@ -39,6 +45,7 @@ def get_stats(db: Session = Depends(get_db), _=Depends(get_current_user)): # Domain distribution domain_rows = ( db.query(OntologyProject.domain, func.count(OntologyProject.id)) + .filter(OntologyProject.id.in_(discoverable_ids)) .group_by(OntologyProject.domain) .all() ) @@ -47,6 +54,7 @@ def get_stats(db: Session = Depends(get_db), _=Depends(get_current_user)): # Status breakdown status_rows = ( db.query(OntologyProject.status, func.count(OntologyProject.id)) + .filter(OntologyProject.id.in_(discoverable_ids)) .group_by(OntologyProject.status) .all() ) @@ -54,10 +62,10 @@ def get_stats(db: Session = Depends(get_db), _=Depends(get_current_user)): return { "data": { - "ontology_count": _safe_count(db, OntologyProject), - "entity_count": _safe_count(db, Entity), - "logic_count": _safe_count(db, LogicRule), - "action_count": _safe_count(db, Action), + "ontology_count": db.query(OntologyProject).filter(OntologyProject.id.in_(discoverable_ids)).count(), + "entity_count": _safe_count(db, Entity, discoverable_ids), + "logic_count": _safe_count(db, LogicRule, discoverable_ids), + "action_count": _safe_count(db, Action, discoverable_ids), "recent_ontologies": recent_list, "domain_counts": domain_counts, "status_counts": status_counts, diff --git a/backend/app/routers/v2/graph.py b/backend/app/routers/v2/graph.py index 7294aafe..c173f9c0 100644 --- a/backend/app/routers/v2/graph.py +++ b/backend/app/routers/v2/graph.py @@ -7,6 +7,8 @@ from app.services.publication.working_copy import OntologyWorkingCopyService from app.services.relation_dedup import dedupe_relation_edges as _dedupe_relation_edges from app.database import SessionLocal +from sqlalchemy.orm import Session +from app.services.ontology_access import GrantDenied, require_project_grant router = APIRouter(dependencies=[Depends(get_current_user)]) @@ -24,12 +26,26 @@ def get_db(): db.close() +def _require_read(ontology_id: str, db: Session = Depends(get_db), actor: User = Depends(get_current_user)): + try: + require_project_grant(db, actor, ontology_id, "read") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + + +def _require_edit(ontology_id: str, db: Session = Depends(get_db), actor: User = Depends(require_editor)): + try: + require_project_grant(db, actor, ontology_id, "edit") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + + class CypherRequest(BaseModel): query: str params: dict = {} -@router.get("/{ontology_id}/graph") +@router.get("/{ontology_id}/graph", dependencies=[Depends(_require_read)]) def get_graph(ontology_id: str, limit: int = 200, label_filter: str | None = None): """返回本体图谱数据 (Neovis.js 兼容格式)""" svc = get_neo4j() @@ -95,7 +111,7 @@ def _sqlite_graph_data(ontology_id: str, limit: int = 200, label_filter: str | N db.close() -@router.get("/{ontology_id}/graph/quality") +@router.get("/{ontology_id}/graph/quality", dependencies=[Depends(_require_read)]) def graph_quality(ontology_id: str): from app.models.entity import Entity from app.models.relation import Relation @@ -146,7 +162,7 @@ def graph_quality(ontology_id: str): db.close() -@router.get("/{ontology_id}/integrations/status") +@router.get("/{ontology_id}/integrations/status", dependencies=[Depends(_require_read)]) def integration_status(ontology_id: str): neo = get_neo4j() neo_available = neo.available @@ -161,7 +177,7 @@ def integration_status(ontology_id: str): } -@router.post("/{ontology_id}/graph/cypher") +@router.post("/{ontology_id}/graph/cypher", dependencies=[Depends(_require_read)]) def run_cypher(ontology_id: str, body: CypherRequest): """执行 Cypher 查询 (只读校验 + 强制 ontology_id 过滤)""" from app.services.v2.graph.cypher_builder import validate_readonly_cypher @@ -180,15 +196,16 @@ def run_cypher(ontology_id: str, body: CypherRequest): return {"results": results, "neo4j_available": True} -@router.get("/{ontology_id}/graph/neighbors/{node_id}") +@router.get("/{ontology_id}/graph/neighbors/{node_id}", dependencies=[Depends(_require_read)]) def get_neighbors(ontology_id: str, node_id: str, depth: int = 1): """查询节点邻居""" svc = get_neo4j() if not svc.available: return {"nodes": [], "edges": [], "neo4j_available": False} query = f""" - MATCH (n)-[r*1..{min(depth, 5)}]-(m) + MATCH path = (n)-[r*1..{min(depth, 5)}]-(m) WHERE elementId(n) = $node_id AND n.ontology_id = $ontology_id + AND all(node IN nodes(path) WHERE node.ontology_id = $ontology_id) RETURN n, r, m LIMIT 100 """ results = svc.run_cypher(query, {"node_id": node_id, "ontology_id": ontology_id}) @@ -203,7 +220,7 @@ class NLQueryRequest(BaseModel): schema: dict = {} -@router.post("/{ontology_id}/graph/ask") +@router.post("/{ontology_id}/graph/ask", dependencies=[Depends(_require_read)]) def nl_query(ontology_id: str, body: NLQueryRequest): """自然语言 → Cypher → 图数据""" from app.services.v2.graph.nl2cypher import NL2CypherService @@ -231,7 +248,7 @@ def nl_query(ontology_id: str, body: NLQueryRequest): # ── 高级图分析 ───────────────────────────────────────────────────────── -@router.get("/{ontology_id}/graph/path") +@router.get("/{ontology_id}/graph/path", dependencies=[Depends(_require_read)]) def graph_path(ontology_id: str, src: str, tgt: str): """两节点间最短路径""" from app.services.v2.graph.graph_analytics import GraphAnalyticsService @@ -239,7 +256,7 @@ def graph_path(ontology_id: str, src: str, tgt: str): return svc.shortest_path(ontology_id, src, tgt) -@router.get("/{ontology_id}/graph/degree/{node_id}") +@router.get("/{ontology_id}/graph/degree/{node_id}", dependencies=[Depends(_require_read)]) def node_degree(ontology_id: str, node_id: str): """查询节点度数(入度 + 出度)""" from app.services.v2.graph.graph_analytics import GraphAnalyticsService @@ -247,7 +264,7 @@ def node_degree(ontology_id: str, node_id: str): return svc.node_degree(ontology_id, node_id) -@router.get("/{ontology_id}/graph/top-nodes") +@router.get("/{ontology_id}/graph/top-nodes", dependencies=[Depends(_require_read)]) def top_nodes(ontology_id: str, limit: int = 10): """返回连接数最多的 Top-N 节点""" from app.services.v2.graph.graph_analytics import GraphAnalyticsService @@ -255,7 +272,7 @@ def top_nodes(ontology_id: str, limit: int = 10): return {"nodes": svc.top_connected_nodes(ontology_id, limit)} -@router.post("/{ontology_id}/graph/sync") +@router.post("/{ontology_id}/graph/sync", dependencies=[Depends(_require_edit)]) def sync_graph(ontology_id: str, current_user: User = Depends(require_editor)): """将 SQLite 实体/关系全量同步到 Neo4j""" from app.database import SessionLocal diff --git a/backend/app/routers/v2/logic_actions.py b/backend/app/routers/v2/logic_actions.py index fb22b32c..509d8cf5 100644 --- a/backend/app/routers/v2/logic_actions.py +++ b/backend/app/routers/v2/logic_actions.py @@ -12,6 +12,7 @@ from app.models.user import User from app.models.v2.logic import OntologyLogicRule, OntologyStateMachine from app.models.v2.action import OntologyActionType, OntologyActionRun +from app.services.ontology_access import GrantDenied, require_project_grant router = APIRouter(dependencies=[Depends(get_current_user)]) @@ -23,6 +24,20 @@ def get_db(): db.close() +def _require_read(ontology_id: str, db: Session = Depends(get_db), actor: User = Depends(get_current_user)): + try: + require_project_grant(db, actor, ontology_id, "read") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + + +def _require_edit(ontology_id: str, db: Session = Depends(get_db), actor: User = Depends(require_editor)): + try: + require_project_grant(db, actor, ontology_id, "edit") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + + # ── Logic Rules ───────────────────────────────────────────────── class LogicRuleCreate(BaseModel): @@ -59,7 +74,7 @@ class LogicTestRequest(BaseModel): parameters: dict = {} -@router.get("/{ontology_id}/logic") +@router.get("/{ontology_id}/logic", dependencies=[Depends(_require_read)]) def list_logic_rules(ontology_id: str, logic_type: str = "", db: Session = Depends(get_db)): q = db.query(OntologyLogicRule).filter(OntologyLogicRule.ontology_id == ontology_id) if logic_type: @@ -73,7 +88,7 @@ def list_logic_rules(ontology_id: str, logic_type: str = "", db: Session = Depen for r in rules] -@router.post("/{ontology_id}/logic", status_code=201) +@router.post("/{ontology_id}/logic", status_code=201, dependencies=[Depends(_require_edit)]) def create_logic_rule(ontology_id: str, body: LogicRuleCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): def _write(): rule = OntologyLogicRule( @@ -87,7 +102,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2logic.create", callback=_write) -@router.put("/{ontology_id}/logic/{rule_id}") +@router.put("/{ontology_id}/logic/{rule_id}", dependencies=[Depends(_require_edit)]) def update_logic_rule(ontology_id: str, rule_id: str, body: LogicRuleCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): rule = db.query(OntologyLogicRule).filter( OntologyLogicRule.id == rule_id, OntologyLogicRule.ontology_id == ontology_id @@ -102,7 +117,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2logic.update", callback=_write) -@router.patch("/{ontology_id}/logic/{rule_id}") +@router.patch("/{ontology_id}/logic/{rule_id}", dependencies=[Depends(_require_edit)]) def patch_logic_rule(ontology_id: str, rule_id: str, body: LogicRulePatch, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): rule = db.query(OntologyLogicRule).filter( OntologyLogicRule.id == rule_id, OntologyLogicRule.ontology_id == ontology_id @@ -121,7 +136,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2logic.patch", callback=_write) -@router.post("/{ontology_id}/logic/{rule_id}/review") +@router.post("/{ontology_id}/logic/{rule_id}/review", dependencies=[Depends(_require_edit)]) def review_logic_rule(ontology_id: str, rule_id: str, body: LogicReviewRequest, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): rule = db.query(OntologyLogicRule).filter( OntologyLogicRule.id == rule_id, OntologyLogicRule.ontology_id == ontology_id @@ -147,7 +162,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2logic.review", callback=_write) -@router.post("/{ontology_id}/logic/{rule_id}/test") +@router.post("/{ontology_id}/logic/{rule_id}/test", dependencies=[Depends(_require_read)]) def test_logic_rule(ontology_id: str, rule_id: str, body: LogicTestRequest, db: Session = Depends(get_db), _=Depends(require_editor)): rule = db.query(OntologyLogicRule).filter( OntologyLogicRule.id == rule_id, OntologyLogicRule.ontology_id == ontology_id @@ -157,7 +172,7 @@ def test_logic_rule(ontology_id: str, rule_id: str, body: LogicTestRequest, db: return {"rule_id": rule.id, **_evaluate_logic_rule(rule, body.row or {}, body.parameters or {})} -@router.delete("/{ontology_id}/logic/{rule_id}") +@router.delete("/{ontology_id}/logic/{rule_id}", dependencies=[Depends(_require_edit)]) def delete_logic_rule(ontology_id: str, rule_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): rule = db.query(OntologyLogicRule).filter( OntologyLogicRule.id == rule_id, OntologyLogicRule.ontology_id == ontology_id @@ -170,7 +185,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2logic.delete", callback=_write) -@router.post("/{ontology_id}/logic/publish") +@router.post("/{ontology_id}/logic/publish", dependencies=[Depends(_require_edit)]) def publish_logic_rules_v2(ontology_id: str, db: Session = Depends(get_db), _=Depends(require_editor)): """Direct tool publication is closed; use the Ontology compiler (P1C).""" raise HTTPException(status_code=403, detail="PUBLICATION_NOT_ENABLED") @@ -197,7 +212,7 @@ def publish_logic_rules_v2(ontology_id: str, db: Session = Depends(get_db), _=De # ── Logic: Discovery ──────────────────────────────────────────── -@router.post("/{ontology_id}/logic/discover") +@router.post("/{ontology_id}/logic/discover", dependencies=[Depends(_require_edit)]) def discover_logic_rules(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): """发现 Logic Rules(同步写入 v2 + v1 表,供前端 LogicTab 读取)""" from app.services.v2.mapping.mapping_service import MappingService @@ -268,7 +283,7 @@ def _result(): # ── State Machines ────────────────────────────────────────────── -@router.get("/{ontology_id}/state-machines") +@router.get("/{ontology_id}/state-machines", dependencies=[Depends(_require_read)]) def list_state_machines(ontology_id: str, db: Session = Depends(get_db)): machines = db.query(OntologyStateMachine).filter( OntologyStateMachine.ontology_id == ontology_id @@ -319,7 +334,7 @@ class ActionReviewRequest(BaseModel): notes: str = "" -@router.get("/{ontology_id}/actions") +@router.get("/{ontology_id}/actions", dependencies=[Depends(_require_read)]) def list_action_types(ontology_id: str, category: str = "", db: Session = Depends(get_db)): q = db.query(OntologyActionType).filter(OntologyActionType.ontology_id == ontology_id) if category: @@ -334,7 +349,7 @@ def list_action_types(ontology_id: str, category: str = "", db: Session = Depend "created_at": a.created_at.isoformat() if a.created_at else None} for a in actions] -@router.post("/{ontology_id}/actions", status_code=201) +@router.post("/{ontology_id}/actions", status_code=201, dependencies=[Depends(_require_edit)]) def create_action_type(ontology_id: str, body: ActionTypeCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): def _write(): act = OntologyActionType( @@ -349,7 +364,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2action.create", callback=_write) -@router.patch("/{ontology_id}/actions/{action_id}") +@router.patch("/{ontology_id}/actions/{action_id}", dependencies=[Depends(_require_edit)]) def patch_action_type(ontology_id: str, action_id: str, body: ActionTypePatch, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): act = db.query(OntologyActionType).filter( OntologyActionType.id == action_id, OntologyActionType.ontology_id == ontology_id @@ -368,7 +383,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2action.patch", callback=_write) -@router.post("/{ontology_id}/actions/{action_id}/review") +@router.post("/{ontology_id}/actions/{action_id}/review", dependencies=[Depends(_require_edit)]) def review_action_type(ontology_id: str, action_id: str, body: ActionReviewRequest, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): act = db.query(OntologyActionType).filter( OntologyActionType.id == action_id, OntologyActionType.ontology_id == ontology_id @@ -399,7 +414,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2action.review", callback=_write) -@router.post("/{ontology_id}/actions/discover") +@router.post("/{ontology_id}/actions/discover", dependencies=[Depends(_require_edit)]) def discover_actions(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): """发现 Actions(同步写入 v2 + v1 表,供前端 ActionsTab 读取)""" from app.services.v2.mapping.mapping_service import MappingService @@ -457,7 +472,7 @@ def _result(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2action.discover", callback=_result) -@router.delete("/{ontology_id}/actions/{action_id}") +@router.delete("/{ontology_id}/actions/{action_id}", dependencies=[Depends(_require_edit)]) def delete_action_type(ontology_id: str, action_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): act = db.query(OntologyActionType).filter( OntologyActionType.id == action_id, OntologyActionType.ontology_id == ontology_id @@ -470,7 +485,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2action.delete", callback=_write) -@router.post("/{ontology_id}/actions/publish") +@router.post("/{ontology_id}/actions/publish", dependencies=[Depends(_require_edit)]) def publish_actions_v2(ontology_id: str, db: Session = Depends(get_db), _=Depends(require_editor)): """Direct tool publication is closed; use the Ontology compiler (P1C).""" raise HTTPException(status_code=403, detail="PUBLICATION_NOT_ENABLED") @@ -497,7 +512,7 @@ def publish_actions_v2(ontology_id: str, db: Session = Depends(get_db), _=Depend # ── Action Runs ───────────────────────────────────────────────── -@router.get("/{ontology_id}/action-runs") +@router.get("/{ontology_id}/action-runs", dependencies=[Depends(_require_read)]) def list_action_runs(ontology_id: str, limit: int = 20, db: Session = Depends(get_db)): runs = db.query(OntologyActionRun).filter( OntologyActionRun.ontology_id == ontology_id @@ -507,7 +522,7 @@ def list_action_runs(ontology_id: str, limit: int = 20, db: Session = Depends(ge "started_at": r.started_at.isoformat() if r.started_at else None} for r in runs] -@router.post("/{ontology_id}/actions/{action_id}/run") +@router.post("/{ontology_id}/actions/{action_id}/run", dependencies=[Depends(_require_edit)]) def run_action_type( ontology_id: str, action_id: str, diff --git a/backend/app/routers/v2/search.py b/backend/app/routers/v2/search.py index 673fc31d..6fcaf540 100644 --- a/backend/app/routers/v2/search.py +++ b/backend/app/routers/v2/search.py @@ -1,13 +1,15 @@ """v2 Search API — 关键词/语义统一搜索""" from __future__ import annotations import json -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel from sqlalchemy import cast, or_, String as SAString from sqlalchemy.orm import Session from app.database import SessionLocal from app.deps import get_current_user from app.models.entity import Entity +from app.models.user import User +from app.services.ontology_access import GrantDenied, require_project_grant router = APIRouter(dependencies=[Depends(get_current_user)]) @@ -20,6 +22,13 @@ def get_db(): db.close() +def _require_read(ontology_id: str, db: Session = Depends(get_db), actor: User = Depends(get_current_user)): + try: + require_project_grant(db, actor, ontology_id, "read") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + + class SearchRequest(BaseModel): query: str mode: str = "keyword" # keyword | semantic @@ -54,7 +63,7 @@ def _sql_keyword_search(db: Session, ontology_id: str, q: str, n: int) -> list[d ] -@router.get("/{ontology_id}/search/keyword") +@router.get("/{ontology_id}/search/keyword", dependencies=[Depends(_require_read)]) def keyword_search( ontology_id: str, q: str = Query(..., description="搜索词"), @@ -71,7 +80,7 @@ def keyword_search( return {"results": results, "chroma_available": True, "query": q} -@router.get("/{ontology_id}/search/semantic") +@router.get("/{ontology_id}/search/semantic", dependencies=[Depends(_require_read)]) def semantic_search( ontology_id: str, q: str = Query(..., description="搜索词"), @@ -87,7 +96,7 @@ def semantic_search( return {"results": results, "chroma_available": True, "query": q} -@router.post("/{ontology_id}/search") +@router.post("/{ontology_id}/search", dependencies=[Depends(_require_read)]) def unified_search(ontology_id: str, body: SearchRequest, db: Session = Depends(get_db)): """统一搜索端点""" from app.services.v2.vector.chroma_service import ChromaService diff --git a/backend/app/services/ontology_access.py b/backend/app/services/ontology_access.py index 6fa7feea..a25b5fdb 100644 --- a/backend/app/services/ontology_access.py +++ b/backend/app/services/ontology_access.py @@ -21,6 +21,8 @@ from sqlalchemy.orm import Session from app.models.ontology_access import OntologyProjectAccessGrant +from app.models.ontology import OntologyProject +from app.models.user import User from app.services.auth_service import get_user_by_id from app.services.governance_audit import enqueue_audit @@ -78,7 +80,7 @@ def _enqueue_audit(db: Session, *, security_domain_id: str, correlation_id: str, def current_grant(db: Session, ontology_id: str, user_id: str) -> dict | None: row = db.execute( sa.text( - "SELECT id, ontology_id, user_id, capabilities, revision, status, created_by, " + "SELECT id, ontology_id, user_id, security_domain_id, capabilities, revision, status, created_by, " "valid_from, valid_until, revoked_by, created_at, updated_at, revoked_at " "FROM ontology_project_access_grants " "WHERE ontology_id = :o AND user_id = :u AND status = 'active'" @@ -226,10 +228,10 @@ def backfill_creator_grants(db: Session) -> dict: def _require_actor(db: Session, actor_id: str, ontology_id: str) -> dict: - grant = current_grant(db, ontology_id, actor_id) - if grant is None or "edit" not in grant["capabilities"]: - raise GrantDenied("actor lacks edit on this ontology") - return grant + actor = get_user_by_id(db, actor_id) + if actor is None: + raise GrantDenied("actor required") + return require_project_grant(db, actor, ontology_id, "edit") def _check_ceiling(role: str, capabilities: list[str]) -> None: @@ -426,14 +428,111 @@ def recover_owner(db: Session, *, ontology_id: str, base_finding_revision: int, def require_project_grant(db: Session, actor, ontology_id: str, capability: str) -> dict: - if actor is None: + if actor is None or not getattr(actor, "id", None) or not getattr(actor, "is_active", False): raise GrantDenied("actor required") - grant = current_grant(db, ontology_id, actor.id) - if grant is None or capability not in grant["capabilities"]: + + if capability not in CAPABILITIES: + raise GrantDenied("unknown capability") + + row = db.execute( + sa.text( + "SELECT g.id, g.ontology_id, g.user_id, g.security_domain_id, g.capabilities, " + "g.revision, g.status, g.created_by, g.valid_from, g.valid_until, g.revoked_by, " + "g.created_at, g.updated_at, g.revoked_at, " + "o.security_domain_id AS ontology_security_domain_id, " + "u.security_domain_id AS actor_security_domain_id, u.is_active AS actor_is_active, " + "u.role AS actor_role " + "FROM ontology_project_access_grants g " + "JOIN ontology_projects o ON o.id = g.ontology_id " + "JOIN users u ON u.id = g.user_id " + "WHERE g.ontology_id = :o AND g.user_id = :u" + ), + { + "o": ontology_id, + "u": actor.id, + }, + ).mappings().one_or_none() + if row is None: + raise GrantDenied("grant not held") + + grant = _row_dict(row) + now = datetime.now(timezone.utc) + valid_from = _as_utc(grant["valid_from"]) + valid_until = _as_utc(grant["valid_until"]) + actor_domain = getattr(actor, "security_domain_id", None) + if ( + not row["actor_is_active"] + or row["actor_security_domain_id"] != actor_domain + or row["ontology_security_domain_id"] != actor_domain + or grant["security_domain_id"] != actor_domain + or grant["status"] != "active" + or (valid_from is not None and valid_from > now) + or (valid_until is not None and now >= valid_until) + or capability not in grant["capabilities"] + or capability not in ROLE_CEILINGS.get(row["actor_role"], ROLE_CEILINGS["viewer"]) + ): raise GrantDenied("grant not held") return grant +def _as_utc(value): + """Treat naive database timestamps as UTC for grant validity checks.""" + if value is None: + return None + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def discoverable_ontology_ids(db: Session, actor): + """Return a SQL selectable of ontology IDs the actor may discover. + + This intentionally remains a selectable so callers can apply the grant + predicate before counting, paginating, or aggregating project rows. + Direct ``read`` grants without ``discover`` remain valid for ID-based + reads and therefore do not appear here. + """ + if actor is None or not getattr(actor, "id", None) or not getattr(actor, "is_active", False): + return sa.select(OntologyProject.id).where(sa.false()) + + capability = "discover" + if capability not in ROLE_CEILINGS.get(getattr(actor, "role", None), ROLE_CEILINGS["viewer"]): + return sa.select(OntologyProject.id).where(sa.false()) + + now = datetime.now(timezone.utc) + grant = OntologyProjectAccessGrant + user = User + ontology = OntologyProject + return ( + sa.select(ontology.id) + .join( + grant, + sa.and_( + grant.ontology_id == ontology.id, + grant.security_domain_id == ontology.security_domain_id, + ), + ) + .join( + user, + sa.and_( + user.id == grant.user_id, + user.security_domain_id == grant.security_domain_id, + ), + ) + .where( + grant.user_id == actor.id, + grant.security_domain_id == getattr(actor, "security_domain_id", None), + user.security_domain_id == getattr(actor, "security_domain_id", None), + user.is_active.is_(True), + grant.status == "active", + sa.or_(grant.valid_from.is_(None), grant.valid_from <= now), + sa.or_(grant.valid_until.is_(None), now < grant.valid_until), + sa.cast(grant.capabilities, sa.Text).like('%"discover"%'), + ) + .distinct() + ) + + def _json_list(values) -> str: import json diff --git a/backend/app/services/v2/graph/graph_analytics.py b/backend/app/services/v2/graph/graph_analytics.py index 76109c24..09634f97 100644 --- a/backend/app/services/v2/graph/graph_analytics.py +++ b/backend/app/services/v2/graph/graph_analytics.py @@ -20,6 +20,7 @@ def get_neighbors(self, ontology_id: str, node_id: str, depth: int = 1) -> dict: query = f""" MATCH path = (n)-[r*1..{depth}]-(m) WHERE elementId(n) = $node_id AND n.ontology_id = $ontology_id + AND all(node IN nodes(path) WHERE node.ontology_id = $ontology_id) RETURN n, relationships(path) AS rels, nodes(path) AS all_nodes LIMIT 100 """ @@ -56,6 +57,7 @@ def shortest_path(self, ontology_id: str, src_id: str, tgt_id: str) -> dict: WHERE elementId(s) = $src AND elementId(t) = $tgt AND s.ontology_id = $ontology_id AND t.ontology_id = $ontology_id MATCH p = shortestPath((s)-[*]-(t)) + WHERE all(node IN nodes(p) WHERE node.ontology_id = $ontology_id) RETURN [n IN nodes(p) | {id: elementId(n), labels: labels(n), name: n.name_cn}] AS path_nodes, length(p) AS path_length """ @@ -80,8 +82,10 @@ def node_degree(self, ontology_id: str, node_id: str) -> dict: query = """ MATCH (n) WHERE elementId(n) = $node_id AND n.ontology_id = $ontology_id - OPTIONAL MATCH (n)-[out]->() - OPTIONAL MATCH ()-[in]->(n) + OPTIONAL MATCH (n)-[out]->(out_neighbor) + WHERE out_neighbor.ontology_id = $ontology_id + OPTIONAL MATCH (in_neighbor)-[in]->(n) + WHERE in_neighbor.ontology_id = $ontology_id RETURN count(DISTINCT out) AS out_degree, count(DISTINCT in) AS in_degree """ try: @@ -102,8 +106,8 @@ def top_connected_nodes(self, ontology_id: str, limit: int = 10) -> list[dict]: if not self._neo4j.available: return [] query = """ - MATCH (n)-[r]-() - WHERE n.ontology_id = $ontology_id + MATCH (n)-[r]-(neighbor) + WHERE n.ontology_id = $ontology_id AND neighbor.ontology_id = $ontology_id RETURN elementId(n) AS node_id, labels(n) AS labels, n.name_cn AS name, diff --git a/backend/tests/agent/test_core_route_registration.py b/backend/tests/agent/test_core_route_registration.py index ddbece72..f7f5f8eb 100644 --- a/backend/tests/agent/test_core_route_registration.py +++ b/backend/tests/agent/test_core_route_registration.py @@ -475,6 +475,8 @@ def test_idempotency_migration_upgrade_downgrade_cycle(): def test_audit_route_rejects_unknown_model_id_422(idem_schema): from app.routers.audit import router as audit_router from app.routers.audit import start_audit # noqa: F401 — route-local fix surface + from app.models.user import User + from app.services.ontology_access import creator_grant session_factory = sessionmaker(bind=create_engine(_scoped_url(idem_schema))) session = session_factory() @@ -491,6 +493,7 @@ def test_audit_route_rejects_unknown_model_id_422(idem_schema): "VALUES (:id, 'p', 'test', 'v0.1', 'created', :by, now(), now(), :domain, 1)" ), {"id": "onto-1", "by": admin_id, "domain": DEFAULT_DOMAIN}) session.commit() + creator_grant(session, "onto-1", session.get(User, admin_id)) finally: session.close() diff --git a/backend/tests/agent/test_import_mutation_closure.py b/backend/tests/agent/test_import_mutation_closure.py index a83d8e6d..42574636 100644 --- a/backend/tests/agent/test_import_mutation_closure.py +++ b/backend/tests/agent/test_import_mutation_closure.py @@ -133,8 +133,13 @@ def test_zz_extraction_rejects_unknown_model_with_stable_error(imports_db): from app.routers import extraction from app.services.auth_service import create_access_token + from app.services.ontology_access import backfill_creator_grants + Session, _ = imports_db actor_id, ontology_id = _seed(imports_db) + # _seed inserts via direct SQL, bypassing create_ontology's creator grant + with Session() as session: + backfill_creator_grants(session) token = create_access_token({"sub": actor_id, "role": "editor"}) headers = {"Authorization": f"Bearer {token}"} diff --git a/backend/tests/agent/test_legacy_route_closure.py b/backend/tests/agent/test_legacy_route_closure.py index be709588..b7bd9294 100644 --- a/backend/tests/agent/test_legacy_route_closure.py +++ b/backend/tests/agent/test_legacy_route_closure.py @@ -151,6 +151,8 @@ def closure_db(): def _seed(closure_db): from app.services.auth_service import hash_password + from app.models.user import User + from app.services.ontology_access import creator_grant Session, _ = closure_db username = "closure-" + uuid.uuid4().hex[:8] @@ -167,6 +169,7 @@ def _seed(closure_db): ), {"id": str(uuid.uuid4()), "name": "Closure " + uuid.uuid4().hex[:8], "creator": actor_id, "domain": DEFAULT_DOMAIN}) ontology_id = session.execute(text("SELECT id FROM ontology_projects WHERE created_by=:c ORDER BY created_at DESC LIMIT 1"), {"c": actor_id}).scalar_one() session.commit() + creator_grant(session, ontology_id, session.get(User, actor_id)) return actor_id, ontology_id diff --git a/backend/tests/agent/test_legacy_runtime_tables.py b/backend/tests/agent/test_legacy_runtime_tables.py index ae82eae0..577fc088 100644 --- a/backend/tests/agent/test_legacy_runtime_tables.py +++ b/backend/tests/agent/test_legacy_runtime_tables.py @@ -160,8 +160,11 @@ def test_audit_and_instance_writes_no_longer_500(legacy_schema): from app.deps import get_db from app.routers import audit as audit_router from app.services.auth_service import create_access_token + from app.services.ontology_access import backfill_creator_grants with Session() as session: + # the ontology was inserted via direct SQL, bypassing create_ontology's creator grant + backfill_creator_grants(session) actor_id = session.execute(text("SELECT id FROM users WHERE username='lr-user'")).scalar_one() token = create_access_token({"sub": actor_id, "role": "admin"}) diff --git a/backend/tests/agent/test_ontology_access_admin.py b/backend/tests/agent/test_ontology_access_admin.py index 02a65eec..d7f7b077 100644 --- a/backend/tests/agent/test_ontology_access_admin.py +++ b/backend/tests/agent/test_ontology_access_admin.py @@ -11,6 +11,7 @@ import hashlib import json import os +from datetime import datetime, timedelta, timezone from pathlib import Path import subprocess import sys @@ -79,7 +80,7 @@ def access_db(): engine.dispose() -def _insert_user(Session, username, role="editor", active=True, domain=True): +def _insert_user(Session, username, role="editor", active=True, domain=True, domain_id=None): domain_clause = ", security_domain_id" if domain else "" domain_value = ", :domain" if domain else "" params = { @@ -90,7 +91,7 @@ def _insert_user(Session, username, role="editor", active=True, domain=True): "active": active, } if domain: - params["domain"] = DEFAULT_DOMAIN + params["domain"] = domain_id or DEFAULT_DOMAIN with Session() as session: session.execute(text( f"INSERT INTO users (id, username, email, password_hash, role, is_active, created_at, updated_at{domain_clause}) " @@ -101,12 +102,12 @@ def _insert_user(Session, username, role="editor", active=True, domain=True): return user_id -def _insert_ontology(Session, name, created_by): +def _insert_ontology(Session, name, created_by, domain_id=None): with Session() as session: session.execute(text( "INSERT INTO ontology_projects (id, name, domain, version, status, created_by, created_at, updated_at, security_domain_id) " "VALUES (:id, :name, 'test', 'v0.1', 'draft', :created_by, now(), now(), :domain)" - ), {"id": str(uuid.uuid4()), "name": name, "created_by": created_by, "domain": DEFAULT_DOMAIN}) + ), {"id": str(uuid.uuid4()), "name": name, "created_by": created_by, "domain": domain_id or DEFAULT_DOMAIN}) ontology_id = session.execute(text("SELECT id FROM ontology_projects WHERE name=:n"), {"n": name}).scalar_one() session.commit() return ontology_id @@ -501,3 +502,166 @@ def override_get_db(): ) assert missing.status_code == 404 test_app.dependency_overrides.clear() + + +def test_zz_require_project_grant_enforces_actor_state_domain_window_and_role(access_db): + from types import SimpleNamespace + + from app.models.user import User + from app.services.ontology_access import ( + GrantDenied, + create_grant, + creator_grant, + require_project_grant, + ) + + Session, _ = access_db + owner = _insert_user(Session, "guard-owner", role="admin") + reader = _insert_user(Session, "guard-reader", role="editor") + target = _insert_user(Session, "guard-target", role="viewer") + ontology_id = _insert_ontology(Session, "Guard ontology", owner) + + with Session() as session: + creator_grant(session, ontology_id, session.get(User, owner)) + create_grant( + session, + ontology_id=ontology_id, + user_id=reader, + capabilities=["discover", "read", "edit", "publish"], + base_revision=0, + actor_id=owner, + ) + reader_user = session.get(User, reader) + assert require_project_grant(session, reader_user, ontology_id, "edit") + + reader_user.role = "viewer" + session.flush() + with pytest.raises(GrantDenied): + require_project_grant(session, reader_user, ontology_id, "edit") + assert require_project_grant(session, reader_user, ontology_id, "read") + + reader_user.role = "editor" + session.execute(text( + "UPDATE ontology_project_access_grants SET valid_from=:valid_from " + "WHERE ontology_id=:o AND user_id=:u" + ), { + "valid_from": datetime.now(timezone.utc) + timedelta(minutes=5), + "o": ontology_id, + "u": reader, + }) + with pytest.raises(GrantDenied): + require_project_grant(session, reader_user, ontology_id, "read") + + session.execute(text( + "UPDATE ontology_project_access_grants SET valid_from=NULL, valid_until=:valid_until " + "WHERE ontology_id=:o AND user_id=:u" + ), { + "valid_until": datetime.now(timezone.utc) - timedelta(seconds=1), + "o": ontology_id, + "u": reader, + }) + with pytest.raises(GrantDenied): + require_project_grant(session, reader_user, ontology_id, "read") + + session.execute(text( + "UPDATE ontology_project_access_grants SET valid_until=NULL WHERE ontology_id=:o AND user_id=:u" + ), {"o": ontology_id, "u": reader}) + with pytest.raises(GrantDenied): + require_project_grant( + session, + SimpleNamespace( + id=reader, + is_active=True, + role="editor", + security_domain_id="00000000-0000-0000-0000-000000000099", + ), + ontology_id, + "read", + ) + + reader_user.is_active = False + session.flush() + with pytest.raises(GrantDenied): + require_project_grant(session, reader_user, ontology_id, "read") + + session.execute(text( + "UPDATE ontology_project_access_grants SET valid_until=:valid_until " + "WHERE ontology_id=:o AND user_id=:u" + ), { + "valid_until": datetime.now(timezone.utc) - timedelta(seconds=1), + "o": ontology_id, + "u": owner, + }) + with pytest.raises(GrantDenied): + create_grant( + session, + ontology_id=ontology_id, + user_id=target, + capabilities=["discover", "read"], + base_revision=0, + actor_id=owner, + ) + + session.execute(text( + "UPDATE ontology_project_access_grants SET valid_until=NULL " + "WHERE ontology_id=:o AND user_id=:u" + ), {"o": ontology_id, "u": owner}) + session.execute(text("UPDATE users SET role='viewer' WHERE id=:u"), {"u": owner}) + with pytest.raises(GrantDenied): + create_grant( + session, + ontology_id=ontology_id, + user_id=target, + capabilities=["discover", "read"], + base_revision=0, + actor_id=owner, + ) + + +def test_zz_discoverable_ontology_ids_filter_capability_window_domain_and_role(access_db): + from app.models.user import User + from app.services.ontology_access import ( + create_grant, + creator_grant, + discoverable_ontology_ids, + ) + + Session, _ = access_db + owner = _insert_user(Session, "discover-owner", role="admin") + viewer = _insert_user(Session, "discover-viewer", role="viewer") + same = _insert_ontology(Session, "Discover same", owner) + no_discover = _insert_ontology(Session, "Discover no capability", owner) + expired = _insert_ontology(Session, "Discover expired", owner) + future = _insert_ontology(Session, "Discover future", owner) + + with Session() as session: + creator_grant(session, same, session.get(User, owner)) + creator_grant(session, no_discover, session.get(User, owner)) + creator_grant(session, expired, session.get(User, owner)) + creator_grant(session, future, session.get(User, owner)) + for oid, capabilities in ( + (same, ["discover", "read"]), + (no_discover, ["read"]), + (expired, ["discover", "read"]), + (future, ["discover", "read"]), + ): + create_grant( + session, + ontology_id=oid, + user_id=viewer, + capabilities=capabilities, + base_revision=0, + actor_id=owner, + ) + session.execute(text( + "UPDATE ontology_project_access_grants SET valid_until=:when " + "WHERE ontology_id=:o AND user_id=:u" + ), {"when": datetime.now(timezone.utc) - timedelta(seconds=1), "o": expired, "u": viewer}) + session.execute(text( + "UPDATE ontology_project_access_grants SET valid_from=:when " + "WHERE ontology_id=:o AND user_id=:u" + ), {"when": datetime.now(timezone.utc) + timedelta(minutes=5), "o": future, "u": viewer}) + actor = session.get(User, viewer) + ids = session.execute(discoverable_ontology_ids(session, actor)).scalars().all() + + assert ids == [same] diff --git a/backend/tests/agent/test_ontology_route_access.py b/backend/tests/agent/test_ontology_route_access.py new file mode 100644 index 00000000..6eda7805 --- /dev/null +++ b/backend/tests/agent/test_ontology_route_access.py @@ -0,0 +1,80 @@ +"""Project grants must protect legacy and classic-v2 ontology API surfaces.""" + +import pytest + +from app.main import app +from app.routers.v2 import graph as graph_v2 +from app.routers.v2 import logic_actions as logic_actions_v2 +from app.routers.v2 import search as search_v2 + + +@pytest.mark.parametrize("path", [ + "/api/v1/ontologies/{id}", + "/api/v1/ontologies/{id}/export?format=json", + "/api/v1/ontologies/{id}/entities", + "/api/v1/ontologies/{id}/logic", + "/api/v1/ontologies/{id}/actions", + "/api/v1/ontologies/{id}/graph", + "/api/v1/ontologies/{id}/files", + "/api/v2/ontologies/{id}/graph", + "/api/v2/ontologies/{id}/search/keyword?q=test", + "/api/v2/ontologies/{id}/logic", + "/api/v2/ontologies/{id}/state-machines", + "/api/v2/ontologies/{id}/actions", +]) +def test_ungranted_editor_cannot_read_ontology_surfaces(client, db, ontology, editor_user, path): + def test_db(): + yield db + + for dependency in (graph_v2.get_db, search_v2.get_db, logic_actions_v2.get_db): + app.dependency_overrides[dependency] = test_db + + token = client.post( + "/api/v1/auth/login", + json={"username": editor_user.username, "password": "editor123"}, + ).json()["data"]["access_token"] + response = client.get( + path.format(id=ontology["id"]), + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 404 + + +@pytest.mark.parametrize("path", [ + "/api/v1/ontologies/{id}/export?format=json", + "/api/v2/ontologies/{id}/search/keyword?q=test", + "/api/v2/ontologies/{id}/logic", +]) +def test_creator_grant_keeps_read_routes_available(client, db, ontology, auth_headers, path): + def test_db(): + yield db + + for dependency in (search_v2.get_db, logic_actions_v2.get_db): + app.dependency_overrides[dependency] = test_db + + response = client.get(path.format(id=ontology["id"]), headers=auth_headers) + assert response.status_code == 200 + + +def _editor_headers(client): + token = client.post( + "/api/v1/auth/login", + json={"username": "editor", "password": "editor123"}, + ).json()["data"]["access_token"] + return {"Authorization": f"Bearer {token}"} + + +def test_ungranted_editor_cannot_write_legacy_entities(client, ontology, editor_user): + response = client.post( + f"/api/v1/ontologies/{ontology['id']}/entities", + json={"name_cn": "越权实体", "name_en": "Intruder"}, + headers=_editor_headers(client), + ) + assert response.status_code == 404 + + +def test_overview_hides_ungranted_ontologies(client, ontology, editor_user): + response = client.get("/api/v1/overview/stats", headers=_editor_headers(client)) + assert response.status_code == 200 + data = response.json()["data"] + assert ontology["id"] not in [item["id"] for item in data["recent_ontologies"]] diff --git a/backend/tests/test_ontology_deletion_with_instances.py b/backend/tests/test_ontology_deletion_with_instances.py index 4dcfd3c0..ecc75074 100644 --- a/backend/tests/test_ontology_deletion_with_instances.py +++ b/backend/tests/test_ontology_deletion_with_instances.py @@ -48,6 +48,17 @@ def schema(): def test_delete_ontology_with_instance_relations_does_not_hit_dependent_data(schema, monkeypatch): engine = create_engine(_scoped_url(schema)) session = sessionmaker(bind=engine)() + try: + _run_delete_scenario(session) + finally: + # an open transaction would block the fixture's DROP SCHEMA forever + session.close() + engine.dispose() + + +def _run_delete_scenario(session): + from app.models.user import User + from app.services.ontology_access import backfill_creator_grants oid, u = "o-del-1", "u-del-1" session.execute(text( @@ -86,10 +97,11 @@ def test_delete_ontology_with_instance_relations_does_not_hit_dependent_data(sch ), {"o": oid}).scalar_one() assert outbox_before > 0 - delete_ontology(ontology_id=oid, db=session) + # rows were inserted via direct SQL, bypassing create_ontology's creator grant + backfill_creator_grants(session) + delete_ontology(ontology_id=oid, db=session, current_user=session.get(User, u)) remaining = session.execute(text( "SELECT count(*) FROM ontology_projects WHERE id = :o" ), {"o": oid}).scalar_one() assert remaining == 0 - session.close() diff --git a/backend/tests/v2/graph/test_analytics.py b/backend/tests/v2/graph/test_analytics.py index 34e83d0b..dc79b532 100644 --- a/backend/tests/v2/graph/test_analytics.py +++ b/backend/tests/v2/graph/test_analytics.py @@ -63,3 +63,39 @@ def test_node_degree_with_result(): result = svc.node_degree("ont-1", "node-1") assert result["in_degree"] == 3 assert result["out_degree"] == 7 + + +def test_graph_traversals_scope_every_returned_node_to_ontology(): + svc, mock_neo4j = make_analytics(available=True) + mock_neo4j.run_cypher.return_value = [] + + svc.get_neighbors("ont-1", "node-1") + neighbors_query = mock_neo4j.run_cypher.call_args.args[0] + assert "all(node IN nodes(path) WHERE node.ontology_id = $ontology_id)" in neighbors_query + + svc.shortest_path("ont-1", "a", "b") + path_query = mock_neo4j.run_cypher.call_args.args[0] + assert "all(node IN nodes(p) WHERE node.ontology_id = $ontology_id)" in path_query + + svc.node_degree("ont-1", "node-1") + degree_query = mock_neo4j.run_cypher.call_args.args[0] + assert "out_neighbor.ontology_id = $ontology_id" in degree_query + assert "in_neighbor.ontology_id = $ontology_id" in degree_query + + svc.top_connected_nodes("ont-1") + top_query = mock_neo4j.run_cypher.call_args.args[0] + assert "neighbor.ontology_id = $ontology_id" in top_query + + +def test_graph_neighbors_route_scopes_every_path_node(monkeypatch): + from app.routers.v2 import graph + + fake_neo4j = MagicMock() + fake_neo4j.available = True + fake_neo4j.run_cypher.return_value = [] + monkeypatch.setattr(graph, "get_neo4j", lambda: fake_neo4j) + + graph.get_neighbors("ont-1", "node-1") + + query = fake_neo4j.run_cypher.call_args.args[0] + assert "all(node IN nodes(path) WHERE node.ontology_id = $ontology_id)" in query diff --git a/backend/tests/v2/test_logic_actions_runtime.py b/backend/tests/v2/test_logic_actions_runtime.py index 4ada5abc..98cc2369 100644 --- a/backend/tests/v2/test_logic_actions_runtime.py +++ b/backend/tests/v2/test_logic_actions_runtime.py @@ -2,6 +2,8 @@ from app.models.action import Action from app.models.logic import LogicRule from app.models.ontology import OntologyProject +from app.models.user import User +from app.services.ontology_access import creator_grant from app.models.v2.logic import OntologyLogicRule from app.models.v2.action import OntologyActionRun, OntologyActionType from app.routers.actions import publish_actions @@ -203,6 +205,8 @@ def test_action_runtime_rejects_missing_required_parameter(db): def test_v1_logic_publish_syncs_v2_status(db): ontology_id = "ont-runtime-5" + actor = User(id="runtime", username="runtime", email="runtime@example.test", password_hash="unused", role="editor") + db.add(actor) db.add(OntologyProject(id=ontology_id, name=ontology_id, domain="test", created_by="runtime", security_domain_id="00000000-0000-0000-0000-000000000001")) db.add(LogicRule( id="logic-v1", @@ -224,9 +228,10 @@ def test_v1_logic_publish_syncs_v2_status(db): status="draft", )) db.commit() + creator_grant(db, ontology_id, actor) with pytest.raises(HTTPException) as exc: - publish_logic_rules(ontology_id, db) + publish_logic_rules(ontology_id, db, actor) assert "PUBLICATION_NOT_ENABLED" in str(exc.value.detail) # direct tool publication is closed; nothing is published assert db.query(LogicRule).filter_by(id="logic-v1").first().status == "draft" @@ -235,6 +240,8 @@ def test_v1_logic_publish_syncs_v2_status(db): def test_v1_action_publish_syncs_v2_status(db): ontology_id = "ont-runtime-6" + actor = User(id="runtime", username="runtime", email="runtime@example.test", password_hash="unused", role="editor") + db.add(actor) db.add(OntologyProject(id=ontology_id, name=ontology_id, domain="test", created_by="runtime", security_domain_id="00000000-0000-0000-0000-000000000001")) db.add(Action( id="action-v1", @@ -254,9 +261,10 @@ def test_v1_action_publish_syncs_v2_status(db): status="draft", )) db.commit() + creator_grant(db, ontology_id, actor) with pytest.raises(HTTPException) as exc: - publish_actions(ontology_id, db) + publish_actions(ontology_id, db, actor) assert "PUBLICATION_NOT_ENABLED" in str(exc.value.detail) # direct tool publication is closed; nothing is published assert db.query(Action).filter_by(id="action-v1").first().status == "draft" From 8faa937cabe56c82caa2d130a8b1ad1c9e77d7ec Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Fri, 13 Nov 2026 08:30:37 +0800 Subject: [PATCH 18/27] fix(upload): stream uploads with a hard size bound File and dataset uploads read the whole body into memory before checking max_upload_mb, so one oversized request could exhaust the worker. Both now read in 1 MiB chunks and stop with 413 as soon as the limit is crossed. A rejected or failed upload no longer leaves a partial file on disk, and deleting a file removes it from disk only after the row deletion commits. Co-Authored-By: Claude Opus 5.5 --- backend/app/routers/files.py | 80 +++++++++++----- backend/app/routers/v2/datasets.py | 33 ++++++- backend/tests/test_graph_files_regressions.py | 95 +++++++++++++++++++ backend/tests/v2/test_upload_limits.py | 31 ++++++ 4 files changed, 210 insertions(+), 29 deletions(-) create mode 100644 backend/tests/test_graph_files_regressions.py diff --git a/backend/app/routers/files.py b/backend/app/routers/files.py index 4c79d6a9..52f7480c 100644 --- a/backend/app/routers/files.py +++ b/backend/app/routers/files.py @@ -1,5 +1,6 @@ import os import uuid +import logging from fastapi import APIRouter, Depends, HTTPException, UploadFile, File from sqlalchemy.orm import Session from app.deps import get_db, get_current_user, require_admin, require_editor @@ -11,6 +12,7 @@ from app.config import settings router = APIRouter() +logger = logging.getLogger(__name__) ALLOWED_TYPES = { "application/pdf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", @@ -77,30 +79,54 @@ async def upload_file( ext = os.path.splitext(file.filename)[1] save_path = os.path.join(upload_dir, f"{file_id}{ext}") - content = await file.read() - if len(content) > settings.max_upload_mb * 1024 * 1024: + max_bytes = settings.max_upload_mb * 1024 * 1024 + declared_size = getattr(file, "size", None) + if declared_size is not None and declared_size > max_bytes: raise HTTPException(413, f"文件超过大小限制 {settings.max_upload_mb}MB") - with open(save_path, "wb") as f: - f.write(content) - - mime = file.content_type or "application/octet-stream" - conversion = convert_document(save_path, mime) - if not conversion.ok: - if os.path.exists(save_path): - os.remove(save_path) - raise HTTPException(422, conversion.error or "文件转换失败") - - db_file = UploadedFile( - id=file_id, - ontology_id=ontology_id, - filename=file.filename, - file_path=save_path, - file_size=len(content), - mime_type=mime, - converted_md=conversion.content, - ) - db.add(db_file); db.commit(); db.refresh(db_file) - return {"data": _file_out(db_file)} + + committed = False + try: + total_bytes = 0 + with open(save_path, "wb") as saved: + while True: + chunk = await file.read(1024 * 1024) + if not chunk: + break + total_bytes += len(chunk) + if total_bytes > max_bytes: + raise HTTPException(413, f"文件超过大小限制 {settings.max_upload_mb}MB") + saved.write(chunk) + + mime = file.content_type or "application/octet-stream" + conversion = convert_document(save_path, mime) + if not conversion.ok: + raise HTTPException(422, conversion.error or "文件转换失败") + + db_file = UploadedFile( + id=file_id, + ontology_id=ontology_id, + filename=file.filename, + file_path=save_path, + file_size=total_bytes, + mime_type=mime, + converted_md=conversion.content, + ) + db.add(db_file) + try: + db.commit() + except Exception: + db.rollback() + raise + committed = True + db.refresh(db_file) + return {"data": _file_out(db_file)} + except Exception: + if not committed and os.path.exists(save_path): + try: + os.remove(save_path) + except OSError: + logger.warning("failed to clean up rejected upload %s", save_path, exc_info=True) + raise @router.delete("/{file_id}", status_code=204) def delete_file(ontology_id: str, file_id: str, db: Session = Depends(get_db), current_user=Depends(require_admin)): @@ -108,6 +134,8 @@ def delete_file(ontology_id: str, file_id: str, db: Session = Depends(get_db), c f = db.query(UploadedFile).filter(UploadedFile.id == file_id, UploadedFile.ontology_id == ontology_id).first() if not f: raise HTTPException(404, "File not found") - if os.path.exists(f.file_path): - os.remove(f.file_path) - db.delete(f); db.commit() + file_path = f.file_path + db.delete(f) + db.commit() + if os.path.exists(file_path): + os.remove(file_path) diff --git a/backend/app/routers/v2/datasets.py b/backend/app/routers/v2/datasets.py index f1811abc..bbc4d273 100644 --- a/backend/app/routers/v2/datasets.py +++ b/backend/app/routers/v2/datasets.py @@ -23,6 +23,30 @@ class DatasetResponse(BaseModel): class Config: from_attributes = True + +_UPLOAD_READ_CHUNK_SIZE = 1024 * 1024 + + +async def _read_upload_bounded( + file: UploadFile, + max_bytes: int, + *, + chunk_size: int = _UPLOAD_READ_CHUNK_SIZE, + error_detail: str | None = None, +) -> bytes: + """Read at most one byte beyond the configured upload limit.""" + content = bytearray() + while len(content) <= max_bytes: + read_size = min(chunk_size, max_bytes - len(content) + 1) + chunk = await file.read(read_size) + if not chunk: + break + if len(content) + len(chunk) > max_bytes: + raise HTTPException(status_code=413, detail=error_detail or "文件超过大小限制") + content.extend(chunk) + return bytes(content) + + @router.post("/upload", status_code=201) async def upload_dataset(file: UploadFile = File(...), db: Session = Depends(get_db), _=Depends(require_editor)): """上传 CSV/Excel 文件,自动创建 raw Dataset + DatasetVersion""" @@ -35,9 +59,12 @@ async def upload_dataset(file: UploadFile = File(...), db: Session = Depends(get if ext not in allowed: raise HTTPException(400, f"不支持的文件类型: .{ext} (允许: {settings.allowed_upload_extensions})") - content = await file.read() - if len(content) > settings.max_upload_mb * 1024 * 1024: - raise HTTPException(413, f"文件超过大小限制 {settings.max_upload_mb}MB") + max_upload_bytes = settings.max_upload_mb * 1024 * 1024 + content = await _read_upload_bounded( + file, + max_upload_bytes, + error_detail=f"文件超过大小限制 {settings.max_upload_mb}MB", + ) # 推断 kind if ext in ("csv", "xlsx", "xls"): kind = "structured" diff --git a/backend/tests/test_graph_files_regressions.py b/backend/tests/test_graph_files_regressions.py new file mode 100644 index 00000000..1c093905 --- /dev/null +++ b/backend/tests/test_graph_files_regressions.py @@ -0,0 +1,95 @@ +import io +import os + +import pytest + +from app.config import settings + + +def test_create_relation_rejects_missing_entity_key_without_500(client, auth_headers, ontology): + response = client.post( + f"/api/v1/ontologies/{ontology['id']}/graph/relations", + json={"source_entity": "missing-target"}, + headers=auth_headers, + ) + + assert response.status_code == 422 + + +def test_create_relation_rejects_entity_from_another_ontology(client, auth_headers, ontology): + first = client.post( + f"/api/v1/ontologies/{ontology['id']}/entities", + json={"name_cn": "First"}, + headers=auth_headers, + ).json()["data"] + other_ontology = client.post( + "/api/v1/ontologies", + json={"name": "Other ontology", "domain": "供应链"}, + headers=auth_headers, + ).json()["data"] + second = client.post( + f"/api/v1/ontologies/{other_ontology['id']}/entities", + json={"name_cn": "Second"}, + headers=auth_headers, + ).json()["data"] + + response = client.post( + f"/api/v1/ontologies/{ontology['id']}/graph/relations", + json={"source_entity": first["id"], "target_entity": second["id"]}, + headers=auth_headers, + ) + + assert response.status_code == 422 + + +def test_upload_conversion_failure_removes_partial_file(client, auth_headers, ontology, tmp_path, monkeypatch): + monkeypatch.setattr(settings, "uploads_dir", str(tmp_path)) + + class FailedConversion: + ok = False + content = None + error = "conversion failed" + + monkeypatch.setattr("app.routers.files.convert_document", lambda *_args: FailedConversion()) + response = client.post( + f"/api/v1/ontologies/{ontology['id']}/files", + files={"file": ("bad.txt", io.BytesIO(b"not usable"), "text/plain")}, + headers=auth_headers, + ) + + assert response.status_code == 422 + assert list(tmp_path.rglob("*")) == [tmp_path / ontology["id"]] + + +def test_delete_file_does_not_remove_disk_file_before_commit( + client, auth_headers, ontology, admin_user, db, tmp_path, monkeypatch, +): + monkeypatch.setattr(settings, "uploads_dir", str(tmp_path)) + uploaded = client.post( + f"/api/v1/ontologies/{ontology['id']}/files", + files={"file": ("ok.txt", io.BytesIO(b"hello"), "text/plain")}, + headers=auth_headers, + ) + assert uploaded.status_code == 201 + # The public response intentionally omits file_path; discover it from the + # temporary ontology directory instead. + paths = [path for path in (tmp_path / ontology["id"]).iterdir() if path.is_file()] + assert len(paths) == 1 + file_path = paths[0] + + from app.routers import files as files_router + + original_commit = type(db).commit + + def failing_commit(self): + raise RuntimeError("commit failed") + + monkeypatch.setattr(type(db), "commit", failing_commit) + try: + with pytest.raises(RuntimeError): + files_router.delete_file( + ontology["id"], uploaded.json()["data"]["id"], db, current_user=admin_user, + ) + finally: + monkeypatch.setattr(type(db), "commit", original_commit) + assert os.path.exists(file_path) diff --git a/backend/tests/v2/test_upload_limits.py b/backend/tests/v2/test_upload_limits.py index 9fab92fe..be5551ed 100644 --- a/backend/tests/v2/test_upload_limits.py +++ b/backend/tests/v2/test_upload_limits.py @@ -2,6 +2,7 @@ import io import pytest +from fastapi import HTTPException from app.config import settings @@ -45,6 +46,36 @@ def test_upload_rejects_oversized_file(datasets_client, admin_user, monkeypatch) assert r.status_code == 413 +@pytest.mark.asyncio +async def test_bounded_upload_read_stops_after_limit_plus_one_byte(monkeypatch): + from app.routers.v2.datasets import _read_upload_bounded + + limit = 8 + payload = b"x" * (limit + 1000) + + class TrackedUpload: + def __init__(self, value): + self.value = value + self.offset = 0 + self.read_sizes = [] + + async def read(self, size=-1): + self.read_sizes.append(size) + end = min(self.offset + size, len(self.value)) + chunk = self.value[self.offset:end] + self.offset = end + return chunk + + upload = TrackedUpload(payload) + + with pytest.raises(HTTPException) as error: + await _read_upload_bounded(upload, limit, chunk_size=4) + + assert error.value.status_code == 413 + assert upload.offset == limit + 1 + assert upload.read_sizes == [4, 4, 1] + + def test_upload_accepts_normal_csv(datasets_client, admin_user): r = datasets_client.post( "/api/v2/datasets/upload", From 8c2756073bbee292b8fcba9e440d5d322bceb9a8 Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Fri, 13 Nov 2026 08:31:28 +0800 Subject: [PATCH 19/27] feat(connections): use saved DB connections as pipeline and refresh sources Pipeline connector nodes and refresh runs now resolve MySQL/Postgres sources from a saved, encrypted Connection instead of inline credentials: - connection/config.py is the single place that decrypts a Connection and builds the SQLAlchemy URL; the router, pipeline runs and refresh polling all go through it. Connection test errors are passed through safe_connection_error so credentials don't leak into responses. - New GET /connections/{id}/resources and .../resources/{resource}/sample let the Builder list tables and preview rows. - Pipeline definitions drop inline SQL credentials on save and hide them on read. Inline Mongo/REST connector nodes are rejected with UNSUPPORTED_INLINE_CONNECTOR. - A pipeline run snapshots each selected table as an immutable DatasetVersion and pins every input version before pulling. - Refresh runs resolve a concrete resource instead of the "__scheduled__" placeholder. Pipeline-target schedules are closed, and legacy rows are disabled at dispatch. - Frontend: ConnectorInspector selects a saved connection and table, and ConnectionsTab can test connections. Co-Authored-By: Claude Opus 5.5 --- backend/app/routers/v2/connections.py | 120 ++++++--- backend/app/routers/v2/pipelines.py | 92 ++++++- backend/app/services/connection/config.py | 96 +++++++ .../app/services/connection/sql_connector.py | 6 +- backend/app/services/v2/dataset_service.py | 8 +- .../app/services/v2/incremental/operations.py | 85 ++++-- .../app/services/v2/incremental/polling.py | 17 +- .../app/services/v2/scheduler/cron_service.py | 25 +- .../services/v2/scheduler/schedule_service.py | 42 ++- backend/app/tasks/v2/pipeline_run.py | 246 ++++++++++++++++-- backend/app/tasks/v2/refresh_tasks.py | 37 ++- backend/openapi-agent.json | 113 ++++++++ .../connection/test_connection_resources.py | 138 ++++++++++ backend/tests/v2/connection/test_scheduler.py | 20 +- .../tests/v2/connection/test_sql_connector.py | 43 +++ .../v2/incremental/test_refresh_schedule.py | 18 +- .../incremental/test_resource_resolution.py | 195 ++++++++++++++ .../tests/v2/pipeline/test_database_source.py | 126 +++++++++ .../tests/v2/pipeline/test_pinned_inputs.py | 104 ++++++++ .../v2/test_pipeline_connector_safety.py | 195 ++++++++++++++ frontend/src/i18n/en.json | 21 +- frontend/src/i18n/zh.json | 21 +- .../inspectors/ConnectorInspector.test.tsx | 114 +++++++- .../builder/inspectors/ConnectorInspector.tsx | 165 +++++++++++- .../connections/ConnectionsTab.test.tsx | 70 +++++ .../pipelines/connections/ConnectionsTab.tsx | 168 +++++++++--- 26 files changed, 2085 insertions(+), 200 deletions(-) create mode 100644 backend/app/services/connection/config.py create mode 100644 backend/tests/v2/connection/test_connection_resources.py create mode 100644 backend/tests/v2/incremental/test_resource_resolution.py create mode 100644 backend/tests/v2/pipeline/test_database_source.py create mode 100644 backend/tests/v2/pipeline/test_pinned_inputs.py create mode 100644 backend/tests/v2/test_pipeline_connector_safety.py create mode 100644 frontend/src/pages/pipelines/connections/ConnectionsTab.test.tsx diff --git a/backend/app/routers/v2/connections.py b/backend/app/routers/v2/connections.py index 5645581c..36e30900 100644 --- a/backend/app/routers/v2/connections.py +++ b/backend/app/routers/v2/connections.py @@ -11,7 +11,7 @@ import json from datetime import datetime, timezone -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel from sqlalchemy.orm import Session from typing import Optional @@ -19,6 +19,12 @@ from app.database import SessionLocal from app.deps import get_current_user, require_editor from app.models.v2.connection import Connection +from app.models.user import User +from app.services.connection.config import ( + ConnectionConfigError, + build_connector_config, + safe_connection_error, +) from app.services.connection.registry import get_connector router = APIRouter(dependencies=[Depends(get_current_user)]) @@ -53,7 +59,7 @@ class Config: # ── 端点 ────────────────────────────────────────────────────── @router.post("", response_model=ConnectionResponse, status_code=201) -def create_connection(body: ConnectionCreate, db: Session = Depends(get_db), _=Depends(require_editor)): +def create_connection(body: ConnectionCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): """创建连接。config 加密后存储。""" from app.services import encryption_service encrypted_config = {"_encrypted": encryption_service.encrypt(json.dumps(body.config))} @@ -63,6 +69,7 @@ def create_connection(body: ConnectionCreate, db: Session = Depends(get_db), _=D kind=body.kind, config=encrypted_config, status="inactive", + created_by=current_user.id, ) db.add(conn) db.commit() @@ -94,35 +101,36 @@ def _build_db_config(raw_config: dict, db_type: str) -> dict: connection_string,这里组装成 SQLAlchemy 可用的连接 URL。 密码中的特殊字符通过 urllib.parse.quote 编码以避免 URL 解析歧义。 """ - from urllib.parse import quote - host = raw_config.get("host", "localhost") - port = raw_config.get("port", "3306" if db_type == "mysql" else "5432") - user = raw_config.get("user", "") - password = raw_config.get("password", "") - database = raw_config.get("database", "") - # 密码/用户名/库名中的特殊字符(如 @ : / # 空格等)必须 URL 编码, - # 否则 SQLAlchemy 的 URL 解析器会将 @ 等视为 URL 结构分隔符而非密码的一部分。 - # host 不编码(IPv6 地址用 [] 括起,需原样保留)。 - scheme = "mysql+pymysql" if db_type == "mysql" else "postgresql" - conn_str = f"{scheme}://{quote(user, safe='')}:{quote(password, safe='')}@{host}:{port}/{quote(database, safe='')}" - config = dict(raw_config) - config["connection_string"] = conn_str - return config + return build_connector_config(db_type, raw_config) + + +def _load_connection_connector( + conn: Connection, + *, + resource: str | None = None, +): + """Build a connector from an encrypted Connection row. + + Keeping this in one helper ensures API tests, pipeline execution, and + refresh workers all resolve the same saved database configuration. + """ + config = build_connector_config(conn.kind, conn.config, resource=resource) + return get_connector(conn.kind, config) @router.post("/test-config") def test_connection_config(body: TestConfigBody, _=Depends(require_editor)): """测试连接配置(无需先创建 Connection,供 Builder 使用)""" try: - cfg = body.config - if body.type in ("mysql", "postgresql", "postgres"): - db_type = "postgres" if "postgres" in body.type else "mysql" - cfg = _build_db_config(cfg, db_type) + cfg = build_connector_config(body.type, body.config) connector = get_connector(body.type, cfg) ok = connector.test_connection() - return {"success": ok} + result = {"success": ok} + if not ok: + result["detail"] = getattr(connector, "last_error", None) or "connection test failed" + return result except Exception as e: - return {"success": False, "detail": str(e)} + return {"success": False, "detail": safe_connection_error(e)} @router.post("/{connection_id}/test") @@ -132,27 +140,67 @@ def test_connection(connection_id: str, db: Session = Depends(get_db), _=Depends if not conn: raise HTTPException(status_code=404, detail="Connection not found") - from app.services import encryption_service - raw = conn.config.get("_encrypted", "") try: - config = json.loads(encryption_service.decrypt(raw)) if raw else conn.config - except Exception: - config = conn.config - - # 与 test-config 一样,ConnectionsTab 存储的是独立字段而非 connection_string - if conn.kind in ("mysql", "postgres"): - config = _build_db_config(config, conn.kind) - - try: - connector = get_connector(conn.kind, config) + connector = _load_connection_connector(conn) ok = connector.test_connection() conn.status = "active" if ok else "error" db.commit() - return {"success": ok, "status": conn.status} + result = {"success": ok, "status": conn.status} + if not ok: + result["detail"] = getattr(connector, "last_error", None) or "connection test failed" + return result except Exception as e: conn.status = "error" db.commit() - return {"success": False, "status": "error", "detail": str(e)} + return {"success": False, "status": "error", "detail": safe_connection_error(e)} + + +@router.get("/{connection_id}/resources") +def list_connection_resources( + connection_id: str, + db: Session = Depends(get_db), + _=Depends(require_editor), +): + """List tables/collections exposed by a saved connection.""" + conn = db.query(Connection).filter(Connection.id == connection_id).first() + if not conn: + raise HTTPException(404, "Connection not found") + try: + connector = _load_connection_connector(conn) + resources = connector.list_resources() + return {"connection_id": connection_id, "resources": resources} + except (ConnectionConfigError, ValueError) as exc: + raise HTTPException(422, detail=str(exc)) from exc + except Exception as exc: + raise HTTPException( + 502, + detail=f"Unable to list resources for connection '{conn.name}': {safe_connection_error(exc)}", + ) from exc + + +@router.get("/{connection_id}/resources/{resource:path}/sample") +def sample_connection_resource( + connection_id: str, + resource: str, + limit: int = Query(20, ge=1, le=1000), + db: Session = Depends(get_db), + _=Depends(require_editor), +): + """Read a bounded sample from one saved connection resource.""" + conn = db.query(Connection).filter(Connection.id == connection_id).first() + if not conn: + raise HTTPException(404, "Connection not found") + try: + connector = _load_connection_connector(conn, resource=resource) + rows = connector.pull_sample(resource, limit=limit) + return {"connection_id": connection_id, "resource": resource, "rows": rows} + except (ConnectionConfigError, ValueError) as exc: + raise HTTPException(422, detail=str(exc)) from exc + except Exception as exc: + raise HTTPException( + 502, + detail=f"Unable to read resource '{resource}': {safe_connection_error(exc)}", + ) from exc class RefreshConfigurationUpdate(BaseModel): diff --git a/backend/app/routers/v2/pipelines.py b/backend/app/routers/v2/pipelines.py index 5441dcd2..12f37211 100644 --- a/backend/app/routers/v2/pipelines.py +++ b/backend/app/routers/v2/pipelines.py @@ -4,6 +4,7 @@ from pydantic import BaseModel from sqlalchemy.orm import Session from typing import Optional +import copy from datetime import datetime, timezone from app.database import SessionLocal from app.deps import get_current_user, require_admin, require_editor @@ -22,6 +23,80 @@ def get_db(): db.close() +_SQL_SOURCE_TYPES = {"mysql", "postgres", "postgresql"} +_UNSUPPORTED_INLINE_SOURCE_TYPES = {"mongo", "mongodb", "rest", "rest_api"} +_UNSUPPORTED_INLINE_CONNECTOR_CODE = "UNSUPPORTED_INLINE_CONNECTOR" + + +def _unsupported_inline_connector_nodes(definition: Optional[dict]) -> list[tuple[str, str]]: + if not isinstance(definition, dict): + return [] + nodes = definition.get("nodes", []) + if not isinstance(nodes, list): + return [] + unsupported = [] + for node in nodes: + if not isinstance(node, dict) or node.get("type") != "connector": + continue + config = node.get("config") + if not isinstance(config, dict): + continue + source_type = str(config.get("source_type", "")).strip().lower() + if source_type in _UNSUPPORTED_INLINE_SOURCE_TYPES: + unsupported.append((str(node.get("id", "")), source_type)) + return unsupported + + +def _reject_unsupported_inline_connectors(definition: Optional[dict]) -> None: + if _unsupported_inline_connector_nodes(definition): + raise HTTPException(status_code=422, detail=_UNSUPPORTED_INLINE_CONNECTOR_CODE) + + +def _sanitize_definition(definition: Optional[dict]) -> Optional[dict]: + """Keep only saved-connection references for SQL connector nodes. + + Credentials come from a saved Connection at run time. Legacy inline + values are not needed by the worker and must never be persisted again. + """ + if definition is None: + return None + sanitized = copy.deepcopy(definition) + nodes = sanitized.get("nodes", []) if isinstance(sanitized, dict) else [] + if not isinstance(nodes, list): + return sanitized + for node in nodes: + if not isinstance(node, dict) or node.get("type") != "connector": + continue + config = node.get("config") + if not isinstance(config, dict): + continue + source_type = str(config.get("source_type", "")).lower() + if source_type not in _SQL_SOURCE_TYPES: + continue + node["config"] = { + key: value for key, value in config.items() + if key in {"source_type", "connection_id", "resource", "table", "sync_mode"} + } + return sanitized + + +def _public_definition(definition: Optional[dict]) -> Optional[dict]: + """Hide credentials in legacy definitions without mutating stored rows.""" + sanitized = _sanitize_definition(definition) + if not isinstance(sanitized, dict): + return sanitized + for node in sanitized.get("nodes", []): + if not isinstance(node, dict) or node.get("type") != "connector": + continue + config = node.get("config") + if not isinstance(config, dict): + continue + source_type = str(config.get("source_type", "")).strip().lower() + if source_type in _UNSUPPORTED_INLINE_SOURCE_TYPES: + node["config"] = {"source_type": source_type} + return sanitized + + # ── Pydantic Models ──────────────────────────────────────────────── class PipelineCreate(BaseModel): @@ -76,6 +151,7 @@ class ValidateResult(BaseModel): @router.post("", response_model=PipelineResponse, status_code=201, dependencies=[Depends(require_editor)]) def create_pipeline(body: PipelineCreate, db: Session = Depends(get_db)): """创建新 Pipeline。支持旧 steps 格式和新 nodes/edges DSL。""" + _reject_unsupported_inline_connectors(body.definition) # 重名校验 existing = db.query(Pipeline).filter( Pipeline.name == body.name, @@ -100,7 +176,7 @@ def create_pipeline(body: PipelineCreate, db: Session = Depends(get_db)): source_dataset_id=body.source_dataset_id, route=inferred_route or "A", # SQLite 该列有 NOT NULL 约束 spec=body.spec or {}, - definition=body.definition, + definition=_sanitize_definition(body.definition), status="draft", branch="main", version=1, @@ -159,8 +235,11 @@ def update_pipeline(pipeline_id: str, body: PipelineUpdate, db: Session = Depend if not pl: raise HTTPException(404, "Pipeline not found") + _reject_unsupported_inline_connectors(body.definition) update_data = body.model_dump(exclude_unset=True) for k, v in update_data.items(): + if k == "definition": + v = _sanitize_definition(v) setattr(pl, k, v) pl.updated_at = datetime.now(timezone.utc) db.commit() @@ -201,6 +280,14 @@ def validate_pipeline(pipeline_id: str, db: Session = Depends(get_db), _=Depends nodes = definition.get("nodes", []) edges = definition.get("edges", []) + for node_id, source_type in _unsupported_inline_connector_nodes(definition): + errors.append({ + "node_id": node_id, + "severity": "error", + "code": _UNSUPPORTED_INLINE_CONNECTOR_CODE, + "message": "MongoDB/REST inline connector nodes are unsupported in Builder; use File Upload in Builder and manage external connections in Data Connections.", + }) + if not nodes: errors.append({"node_id": "", "severity": "error", "message": "Pipeline 至少需要一个节点。"}) @@ -320,6 +407,7 @@ def run_pipeline(pipeline_id: str, db: Session = Depends(get_db), _=Depends(requ pl = db.query(Pipeline).filter(Pipeline.id == pipeline_id).first() if not pl: raise HTTPException(404, "Pipeline not found") + _reject_unsupported_inline_connectors(pl.definition) pl.status = "running" run = PipelineRun(pipeline_id=pipeline_id, status="pending", started_at=datetime.now(timezone.utc)) @@ -453,7 +541,7 @@ def _format_pipeline(pl: Pipeline) -> dict: "source_dataset_id": pl.source_dataset_id, "route": pl.route, "spec": pl.spec or {}, - "definition": pl.definition, + "definition": _public_definition(pl.definition), "status": pl.status or "draft", "branch": pl.branch or "main", "version": pl.version or 1, diff --git a/backend/app/services/connection/config.py b/backend/app/services/connection/config.py new file mode 100644 index 00000000..c865d421 --- /dev/null +++ b/backend/app/services/connection/config.py @@ -0,0 +1,96 @@ +"""Shared connection configuration handling. + +Connection records keep their credentials inside an encrypted JSON envelope. +Every runtime path that opens a source database goes through this module so a +saved connection and a refresh worker build the same SQLAlchemy URL. +""" +from __future__ import annotations + +import json +from typing import Any, Mapping +from urllib.parse import quote + + +SQL_KINDS = {"mysql", "postgres", "postgresql"} + + +class ConnectionConfigError(ValueError): + """A saved connection could not be converted into runtime configuration.""" + + +def decrypt_connection_config(raw_config: Mapping[str, Any] | None) -> dict[str, Any]: + """Return a runtime config from either plaintext or an encrypted envelope. + + Plaintext is accepted for compatibility with old fixtures and direct + connector callers. Persisted API writes always use the encrypted form. + """ + raw = dict(raw_config or {}) + encrypted = raw.get("_encrypted") + if not encrypted: + return raw + try: + from app.services import encryption_service + + decoded = json.loads(encryption_service.decrypt(str(encrypted))) + except Exception as exc: + raise ConnectionConfigError("connection configuration could not be decrypted") from exc + if not isinstance(decoded, dict): + raise ConnectionConfigError("connection configuration must be a JSON object") + return decoded + + +def _sql_url(raw_config: Mapping[str, Any], kind: str) -> str: + existing = raw_config.get("connection_string") + if existing: + return str(existing) + + host = str(raw_config.get("host") or "localhost") + port = raw_config.get("port") or (3306 if kind == "mysql" else 5432) + try: + port_value = int(port) + except (TypeError, ValueError) as exc: + raise ConnectionConfigError("database port must be a number") from exc + if not 1 <= port_value <= 65535: + raise ConnectionConfigError("database port must be between 1 and 65535") + + user = str(raw_config.get("user") or "") + password = str(raw_config.get("password") or "") + database = str(raw_config.get("database") or "") + if not user or not database: + raise ConnectionConfigError("database user and database name are required") + + scheme = "mysql+pymysql" if kind == "mysql" else "postgresql+psycopg2" + # Keep the host as supplied (including bracketed IPv6 literals), while + # encoding URL components that may contain @, :, /, #, or whitespace. + return ( + f"{scheme}://{quote(user, safe='')}:{quote(password, safe='')}" + f"@{host}:{port_value}/{quote(database, safe='')}" + ) + + +def build_connector_config( + kind: str, + raw_config: Mapping[str, Any] | None, + *, + resource: str | None = None, +) -> dict[str, Any]: + """Resolve persisted config and add the runtime DSN for SQL sources.""" + normalized_kind = "postgres" if kind == "postgresql" else kind + config = decrypt_connection_config(raw_config) + if normalized_kind in SQL_KINDS: + config["connection_string"] = _sql_url(config, normalized_kind) + if resource is not None: + config["resource"] = resource + return config + + +def safe_connection_error(exc: BaseException) -> str: + """Return useful connector detail while redacting common URL secrets.""" + message = str(exc) or exc.__class__.__name__ + # SQLAlchemy normally masks passwords, but drivers occasionally include a + # raw DSN in their exception. Avoid leaking password material in API + # responses and logs when that happens. + import re + + message = re.sub(r"(://[^:/@]+:)[^@]+(@)", r"\1***\2", message) + return message[:500] diff --git a/backend/app/services/connection/sql_connector.py b/backend/app/services/connection/sql_connector.py index c15b0abe..7b54171e 100644 --- a/backend/app/services/connection/sql_connector.py +++ b/backend/app/services/connection/sql_connector.py @@ -8,6 +8,7 @@ from sqlalchemy import create_engine, inspect, text from app.services.connection.base import ConnectorBase, DeltaPage, records_to_delta_page +from app.services.connection.config import safe_connection_error from app.schemas.refresh import SourceCursor # 合法 SQL 标识符:字母、数字、下划线、点(schema.table);禁止空格/分号/引号/注释等 @@ -36,6 +37,7 @@ class SQLConnector(ConnectorBase): def __init__(self, config: dict): self._config = config self._engine = None + self.last_error: str | None = None def _get_engine(self): if self._engine is None: @@ -50,8 +52,10 @@ def test_connection(self) -> bool: try: with self._get_engine().connect() as conn: conn.execute(text("SELECT 1")) + self.last_error = None return True - except Exception: + except Exception as exc: + self.last_error = safe_connection_error(exc) return False def list_resources(self) -> list[str]: diff --git a/backend/app/services/v2/dataset_service.py b/backend/app/services/v2/dataset_service.py index 14431bd6..d1cab5c8 100644 --- a/backend/app/services/v2/dataset_service.py +++ b/backend/app/services/v2/dataset_service.py @@ -110,11 +110,11 @@ def preview( self, dataset_id: str, version_no: int, - limit: int = 100, + limit: int | None = 100, *, version_id: str | None = None, ) -> list[dict]: - """CSV/JSON 数据预览。无需 DuckDB, 纯 Python 处理。""" + """CSV/JSON 数据预览。``limit=None`` 返回完整版本。""" query = self._db.query(DatasetVersion).filter(DatasetVersion.dataset_id == dataset_id) if version_id is not None: query = query.filter(DatasetVersion.id == version_id) @@ -156,7 +156,7 @@ def preview( if i < len(headers): row_dict[headers[i]] = val if val is not None else "" rows_list.append(row_dict) - if len(rows_list) >= limit: + if limit is not None and len(rows_list) >= limit: break wb.close() return rows_list @@ -168,7 +168,7 @@ def preview( reader = csv.DictReader(io.StringIO(text)) rows = [] for i, row in enumerate(reader): - if i >= limit: + if limit is not None and i >= limit: break rows.append(dict(row)) return rows diff --git a/backend/app/services/v2/incremental/operations.py b/backend/app/services/v2/incremental/operations.py index 69e6abbb..cc86601c 100644 --- a/backend/app/services/v2/incremental/operations.py +++ b/backend/app/services/v2/incremental/operations.py @@ -17,6 +17,7 @@ from sqlalchemy.orm import Session from app.models.v2.connection import Connection +from app.models.v2.dataset import Dataset from app.models.v2.pipeline import PipelineRunInput from app.models.v2.refresh import ( RefreshDeadLetter, @@ -39,12 +40,14 @@ collect_refresh_operability, ) from app.services.v2.scheduler.schedule_service import ( - DEFAULT_RESOURCE, ScheduleRequest, upsert_refresh_schedule, ) +_LEGACY_RESOURCE_SENTINEL = "__scheduled__" + + class RefreshRunView(BaseModel): """Non-secret, serializable view of a durable refresh run.""" @@ -147,25 +150,76 @@ def _source_state( ).scalars().first() -def _source_resource( - db: Session, *, source_id: str, resource: str | None = None, -) -> str: - if resource: - return resource - state = _source_state(db, source_id=source_id) - if state is not None: - return state.resource +def resolve_refresh_resource(db: Session, *, source_id: str) -> str: + """Resolve one concrete resource owned by a saved connection. + + The resolver deliberately evaluates one precedence level at a time. A + later source of truth cannot hide an ambiguity in an earlier one, and the + historical ``__scheduled__`` placeholder is never a candidate. + """ connection = db.get(Connection, source_id) if connection is None: raise RefreshError("SOURCE_NOT_FOUND", f"no source connection {source_id}") + + def candidates(values: list[object]) -> list[str]: + return sorted({str(value).strip() for value in values if value is not None and str(value).strip()}) + # Fail closed: an `_encrypted` config that cannot be decrypted must - # never be silently treated as "no resource hint configured" -- that - # would resolve to DEFAULT_RESOURCE and let a manual refresh trigger - # create a RefreshSourceState/RefreshRun for the wrong resource instead - # of surfacing the broken configuration. Credentials are never - # accepted from this request path; only the resource hint is read. + # never be silently treated as "no resource hint configured". Credentials + # are never accepted from this request path; only the resource hint is + # read. raw = _connection_configuration(connection) - return str(raw.get("resource") or raw.get("table") or DEFAULT_RESOURCE) + configured = candidates([raw.get("resource"), raw.get("table")]) + configured = [value for value in configured if value != _LEGACY_RESOURCE_SENTINEL] + if len(configured) > 1: + raise RefreshError( + "REFRESH_RESOURCE_AMBIGUOUS", + f"connection {source_id} has multiple configured resources", + ) + if configured: + return configured[0] + + state_resources = candidates(db.execute( + select(RefreshSourceState.resource).where( + RefreshSourceState.source_id == source_id, + RefreshSourceState.resource != _LEGACY_RESOURCE_SENTINEL, + ).distinct() + ).scalars().all()) + if len(state_resources) > 1: + raise RefreshError( + "REFRESH_RESOURCE_AMBIGUOUS", + f"connection {source_id} has multiple refresh resource states", + ) + if state_resources: + return state_resources[0] + + dataset_resources = candidates(db.execute( + select(Dataset.name).where( + Dataset.source_connection_id == source_id, + Dataset.name != _LEGACY_RESOURCE_SENTINEL, + ).distinct() + ).scalars().all()) + if len(dataset_resources) > 1: + raise RefreshError( + "REFRESH_RESOURCE_AMBIGUOUS", + f"connection {source_id} has multiple source datasets", + ) + if dataset_resources: + return dataset_resources[0] + + raise RefreshError( + "REFRESH_RESOURCE_REQUIRED", + f"connection {source_id} has no concrete refresh resource", + ) + + +def _source_resource( + db: Session, *, source_id: str, resource: str | None = None, +) -> str: + """Backwards-compatible explicit-resource adapter for API callers.""" + if resource: + return resource + return resolve_refresh_resource(db, source_id=source_id) def _connection_configuration(connection: Connection) -> dict[str, Any]: @@ -624,6 +678,7 @@ def get_refresh_health( __all__ = [ "RefreshRunView", "RefreshStatus", "RefreshScheduleView", "get_refresh_schedule", + "resolve_refresh_resource", "trigger_refresh", "cancel_refresh_run", "replay_refresh_run", "get_last_successful_refresh_run", "get_refresh_status", "get_refresh_health", "schedule_view", "update_refresh_schedule", diff --git a/backend/app/services/v2/incremental/polling.py b/backend/app/services/v2/incremental/polling.py index 1420936c..e060cfd2 100644 --- a/backend/app/services/v2/incremental/polling.py +++ b/backend/app/services/v2/incremental/polling.py @@ -43,6 +43,7 @@ ) from app.services.connection.base import DeltaPage, records_to_delta_page from app.services.connection.registry import get_connector +from app.services.connection.config import build_connector_config, decrypt_connection_config from app.services.v2.incremental.contract import ( _cursor_to_json, _lock_or_create_source_state, @@ -128,22 +129,16 @@ def _connector_for_source( def _source_configuration(state: RefreshSourceState | None, connection: Connection | None) -> dict: - raw = dict(connection.config or {}) if connection is not None else {} - encrypted = raw.get("_encrypted") - if encrypted: - try: - from app.services import encryption_service - - decrypted = encryption_service.decrypt(encrypted) - raw = json.loads(decrypted) - except Exception: - logger.warning("unable to decrypt refresh source configuration", exc_info=True) + raw = decrypt_connection_config(connection.config if connection is not None else {}) config = dict(raw) if state is not None and state.configuration: # Refresh revisions carry source-owned cursor/policy settings while # credentials and connection details remain on Connection. config.update(state.configuration) - return config + resource = state.resource if state is not None else config.get("resource") + if connection is None: + return config + return build_connector_config(connection.kind, config, resource=resource) def _as_utc(value: datetime | None) -> datetime | None: diff --git a/backend/app/services/v2/scheduler/cron_service.py b/backend/app/services/v2/scheduler/cron_service.py index 25e5443b..94f3fdb6 100644 --- a/backend/app/services/v2/scheduler/cron_service.py +++ b/backend/app/services/v2/scheduler/cron_service.py @@ -72,27 +72,10 @@ def schedule_connection_sync(self, connection_id: str, cron_expr: str, *, db: "S def schedule_pipeline_run(self, pipeline_id: str, cron_expr: str, *, db: "Session | None" = None, **schedule_kwargs) -> dict: - """为 Pipeline 注册定时运行任务。语义同 schedule_connection_sync。""" - cron_params = self.parse_cron(cron_expr) - if db is None: - logger.info(f"Pipeline {pipeline_id} cron 校验通过(未持久化): {cron_expr}") - return { - "pipeline_id": pipeline_id, - "cron": cron_expr, - "celery_crontab": cron_params, - "status": "validated", - } - - schedule = self._persist_schedule(db, target_type="pipeline", target_id=pipeline_id, - cron_expr=cron_expr, **schedule_kwargs) - logger.info(f"Pipeline {pipeline_id} 调度已持久化: {cron_expr}") - return { - "pipeline_id": pipeline_id, - "cron": cron_expr, - "celery_crontab": cron_params, - "status": "scheduled", - "next_due_at": schedule.next_due_at.isoformat() if schedule.next_due_at else None, - } + """Reject unsupported pipeline-target refresh schedules.""" + raise ValueError( + "PIPELINE_SCHEDULE_UNSUPPORTED: pipeline-target schedules are not supported" + ) @staticmethod def _persist_schedule(db: "Session", *, target_type: str, target_id: str, cron_expr: str, diff --git a/backend/app/services/v2/scheduler/schedule_service.py b/backend/app/services/v2/scheduler/schedule_service.py index f868e917..2e66b959 100644 --- a/backend/app/services/v2/scheduler/schedule_service.py +++ b/backend/app/services/v2/scheduler/schedule_service.py @@ -20,6 +20,7 @@ from __future__ import annotations import uuid +import logging from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import Callable, Mapping, Sequence @@ -38,6 +39,7 @@ ) _cron_service = CronService() +logger = logging.getLogger(__name__) # ScheduleRequest.target_type is the API-facing vocabulary ("connection" / # "pipeline"); RefreshSchedule.target_type persists the same vocabulary the @@ -47,11 +49,12 @@ _STORED_TARGET_TYPE = {"connection": "source", "pipeline": "pipeline"} _TASK_NAME_BY_STORED_TARGET_TYPE = {"source": "refresh.connection", "pipeline": "refresh.pipeline"} -# A connection/pipeline's schedule does not yet select a specific resource -# (multi-resource fan-out is a later task's concern); scheduled runs are -# scoped to this fixed placeholder resource until then. +# Retained as an import-compatible fixture value for older callers. New +# schedule dispatch must resolve a concrete resource before creating a run. DEFAULT_RESOURCE = "__scheduled__" +PIPELINE_SCHEDULE_UNSUPPORTED = "PIPELINE_SCHEDULE_UNSUPPORTED" + _MAX_RETRY_ATTEMPTS = 20 _DEFAULT_MAX_ATTEMPTS = 3 _MAX_RETRY_BACKOFF_SECONDS = 3600 @@ -131,6 +134,10 @@ def _validate_retry_policy(retry_policy: Mapping[str, object] | None) -> None: def _validate_request(request: ScheduleRequest) -> ZoneInfo: if request.target_type not in _STORED_TARGET_TYPE: raise ValueError(f"unsupported target_type: {request.target_type!r}") + if request.target_type == "pipeline": + raise ValueError( + f"{PIPELINE_SCHEDULE_UNSUPPORTED}: pipeline-target schedules are not supported" + ) if not request.target_id: raise ValueError("target_id is required") if not _cron_service.validate_cron(request.cron_expr): @@ -315,9 +322,34 @@ def _dispatch_one_schedule( db.commit() return None + if schedule.target_type == "pipeline": + # Rows written before pipeline-target scheduling was closed remain + # readable, but must never be turned into a polling run whose + # ``source_id`` is actually a pipeline id. + schedule.enabled = False + schedule.next_due_at = None + logger.warning( + "Skipping unsupported pipeline refresh schedule %s (%s): %s", + schedule.id, schedule.target_id, PIPELINE_SCHEDULE_UNSUPPORTED, + ) + db.commit() + return None + source_id = schedule.target_id - resource = DEFAULT_RESOURCE - task_name = _TASK_NAME_BY_STORED_TARGET_TYPE[schedule.target_type] + from app.services.v2.incremental.operations import resolve_refresh_resource + from app.schemas.refresh import RefreshError + + try: + resource = resolve_refresh_resource(db, source_id=source_id) + except RefreshError as exc: + logger.warning( + "Skipping refresh schedule %s for source %s: %s", + schedule.id, source_id, exc.reason_code, + ) + schedule.next_due_at = _next_fire_utc(schedule.cron_expression, tz, max(due_at, now)) + db.commit() + return None + task_name = _TASK_NAME_BY_STORED_TARGET_TYPE["source"] idempotency_key = f"schedule:{schedule.id}:{due_at.isoformat()}" run = _lock_schedule_occurrence( db, source_id=source_id, resource=resource, idempotency_key=idempotency_key, diff --git a/backend/app/tasks/v2/pipeline_run.py b/backend/app/tasks/v2/pipeline_run.py index 7326181d..5574d1b6 100644 --- a/backend/app/tasks/v2/pipeline_run.py +++ b/backend/app/tasks/v2/pipeline_run.py @@ -3,11 +3,14 @@ from datetime import datetime, timezone import logging from pathlib import Path -from typing import Sequence +from typing import Mapping, Sequence import uuid logger = logging.getLogger(__name__) +from app.services.connection.config import build_connector_config, safe_connection_error +from app.services.connection.registry import get_connector + def get_celery_app(): try: @@ -202,6 +205,7 @@ def _find_dataset_for_file(db, filename: str): def _collect_sources(db, pl) -> list[dict]: + from app.models.v2.connection import Connection from app.models.v2.dataset import Dataset sources: list[dict] = [] @@ -209,7 +213,29 @@ def _collect_sources(db, pl) -> list[dict]: for node in definition.get("nodes") or []: if node.get("type") != "connector": continue - for file_info in (node.get("config") or {}).get("files", []) or []: + config = node.get("config") or {} + source_type = str(config.get("source_type") or "").lower() + if source_type in {"mysql", "postgres", "postgresql"}: + connection_id = str(config.get("connection_id") or "") + resource = str(config.get("resource") or config.get("table") or "") + if not connection_id: + raise ValueError("Database connector must select a saved connection") + if not resource: + raise ValueError("Database connector must select a table") + connection = db.get(Connection, connection_id) + if connection is None: + raise ValueError(f"Saved connection not found: {connection_id}") + sources.append({ + "connection_id": connection.id, + "resource": resource, + "filename": resource, + "route": "A", + "kind": "structured", + "dataset_id": None, + }) + continue + + for file_info in config.get("files", []) or []: filename = file_info.get("name") or file_info.get("filename") or "" dataset_id = file_info.get("dataset_id") ds = db.query(Dataset).filter(Dataset.id == dataset_id).first() if dataset_id else None @@ -237,13 +263,146 @@ def _collect_sources(db, pl) -> list[dict]: seen: set[str] = set() unique_sources = [] for source in sources: - if source["dataset_id"] in seen: + key = source.get("dataset_id") or ( + f"connection:{source.get('connection_id')}:{source.get('resource')}" + ) + if key in seen: continue - seen.add(source["dataset_id"]) + seen.add(key) unique_sources.append(source) return unique_sources +def _existing_database_dataset(db, connection_id: str, resource: str): + from app.models.v2.dataset import Dataset + + return db.query(Dataset).filter( + Dataset.source_connection_id == connection_id, + Dataset.name == resource, + ).order_by(Dataset.created_at.desc()).first() + + +def _pinned_database_version(db, dataset_id: str, input_dataset_version_ids) -> object | None: + if not input_dataset_version_ids: + return None + from app.models.v2.dataset import DatasetVersion + + for version_id in input_dataset_version_ids: + version = db.get(DatasetVersion, version_id) + if version is not None and version.dataset_id == dataset_id: + return version + return None + + +def _resolve_pinned_input_versions(db, sources: list[dict], input_dataset_version_ids) -> dict[int, object]: + """Resolve one immutable input version for every source before any pull. + + A refresh-triggered pipeline must not silently fill a missing source pin + from a current/latest dataset version. SQL sources may not have a + materialized Dataset until their first unpinned run, so their candidate + Dataset rows are resolved by the saved connection/resource identity. + """ + if not input_dataset_version_ids: + return {} + + from app.models.v2.dataset import Dataset, DatasetVersion + + requested_ids = list(input_dataset_version_ids) + versions = db.query(DatasetVersion).filter(DatasetVersion.id.in_(set(requested_ids))).all() + versions_by_id = {version.id: version for version in versions} + resolved: dict[int, object] = {} + + for source in sources: + dataset_ids: set[str] = set() + if source.get("dataset_id"): + dataset_ids.add(source["dataset_id"]) + elif source.get("connection_id"): + datasets = db.query(Dataset).filter( + Dataset.source_connection_id == source["connection_id"], + Dataset.name == source["resource"], + ).all() + dataset_ids.update(dataset.id for dataset in datasets) + + matches = [ + versions_by_id[version_id] + for version_id in requested_ids + if version_id in versions_by_id + and versions_by_id[version_id].dataset_id in dataset_ids + ] + if len(matches) != 1: + source_name = source.get("resource") or source.get("filename") or "unknown" + raise ValueError( + "PINNED_INPUTS_INCOMPLETE: source " + f"{source_name!r} requires exactly one pinned dataset version" + ) + resolved[id(source)] = matches[0] + + return resolved + + +def _snapshot_database_source( + db, svc, source: dict, input_dataset_version_ids, selected_version=None, +) -> object: + """Materialize a selected SQL table as an immutable DatasetVersion. + + A refresh-triggered run can provide an exact pinned version; in that case + the live table is never read again. Manual runs without a pin read the + selected resource once and persist a new snapshot before Route A loads it. + """ + from app.models.v2.connection import Connection + from app.models.v2.dataset import Dataset + + connection = db.get(Connection, source["connection_id"]) + if connection is None: + raise ValueError(f"Saved connection not found: {source['connection_id']}") + dataset = _existing_database_dataset(db, connection.id, source["resource"]) + if selected_version is not None: + # A pinned SQL version is already immutable. Reuse it directly and + # recover the Dataset identity without touching the live connector. + if dataset is None: + dataset = db.get(Dataset, selected_version.dataset_id) + if ( + dataset is None + or dataset.id != selected_version.dataset_id + or dataset.source_connection_id != connection.id + or dataset.name != source["resource"] + ): + raise ValueError( + "PINNED_INPUTS_INCOMPLETE: pinned version does not belong to " + f"{connection.id}/{source['resource']}" + ) + source["dataset_id"] = dataset.id + return selected_version + if dataset is not None: + pinned = _pinned_database_version(db, dataset.id, input_dataset_version_ids) + if pinned is not None: + source["dataset_id"] = dataset.id + return pinned + + config = build_connector_config( + connection.kind, connection.config, resource=source["resource"], + ) + # A pipeline table selector is authoritative. Do not allow an optional + # saved query to redirect it to an arbitrary statement. + config.pop("query", None) + connector = get_connector(connection.kind, config) + rows = connector.pull_full(source["resource"]) + if isinstance(rows, Mapping): + rows = [dict(rows)] + else: + rows = [dict(row) for row in (rows or [])] + + if dataset is None: + dataset = svc.create_dataset( + name=source["resource"], kind="structured", connection_id=connection.id, + ) + version = svc.create_version( + dataset.id, _safe_csv_bytes(rows), rowcount=len(rows), + ) + source["dataset_id"] = dataset.id + return version + + def _load_source_rows( db, svc, @@ -273,8 +432,14 @@ def _load_source_rows( }] if version is None: return [] + # A database snapshot is already a bounded, immutable input selected by + # the pipeline. Loading only the preview cap would silently drop rows + # from the pipeline, so request the complete version for DB sources while + # retaining the existing preview cap for file sources. + preview_limit = None if source.get("connection_id") else limit return svc.preview( - source["dataset_id"], version.version_no, limit=limit, version_id=version.id, + source["dataset_id"], version.version_no, + limit=preview_limit, version_id=version.id, ) @@ -375,6 +540,21 @@ def _save_curated_outputs(db, svc, pl, source: dict, data: list[dict], ctx, mult return [_save_curated_dataset(db, svc, pl, source, data, ctx, multi_source)] +def _set_pipeline_status_if_latest(db, pipeline, run, status: str) -> bool: + """Update the pipeline summary only for the newest deterministic run.""" + from app.models.v2.pipeline import PipelineRun + + latest = db.query(PipelineRun).filter( + PipelineRun.pipeline_id == run.pipeline_id, + ).order_by( + PipelineRun.created_at.desc(), PipelineRun.id.desc(), + ).first() + if latest is None or latest.id != run.id: + return False + pipeline.status = status + return True + + def pipeline_run_task( pipeline_id: str, run_id: str, input_dataset_version_ids: Sequence[str] | None = None, @@ -382,20 +562,18 @@ def pipeline_run_task( """Pipeline 执行任务 — 支持 DAG 编译 + 节点状态追踪""" from app.database import SessionLocal from app.models.v2.pipeline import Pipeline, PipelineRun, PipelineRunInput - from app.models.v2.dataset import DatasetVersion from app.services.v2.pipeline.base import PipelineContext from app.services.v2.pipeline.dag_compiler import compile_definition from app.services.v2.dataset_service import DatasetService from app.services.v2.incremental.polling import select_pinned_dataset_version db = SessionLocal() + run = None + pl = None try: run = db.query(PipelineRun).filter(PipelineRun.id == run_id).first() if not run: return - run.status = "running" - run.started_at = datetime.now(timezone.utc) - db.commit() pl = db.query(Pipeline).filter(Pipeline.id == pipeline_id).first() if not pl: @@ -404,6 +582,11 @@ def pipeline_run_task( db.commit() return + run.status = "running" + run.started_at = datetime.now(timezone.utc) + _set_pipeline_status_if_latest(db, pl, run, "running") + db.commit() + # ── DAG 编译 ────────────────────────────────────────────── definition = pl.definition plan = compile_definition(definition) @@ -433,30 +616,40 @@ def set_node_status(nid: str, status: str): else: input_dataset_version_ids = list(input_dataset_version_ids) lineage_version_ids = list(input_dataset_version_ids) + pinned_versions = _resolve_pinned_input_versions( + db, sources, input_dataset_version_ids, + ) if input_dataset_version_ids: run.dataset_version_id = input_dataset_version_ids[0] db.commit() transform_route, runtime_spec = _pipeline_runtime_config(pl) - if sources and not pl.source_dataset_id: - pl.source_dataset_id = sources[0]["dataset_id"] - db.commit() - outputs = [] multi_source = len(sources) > 1 for source in sources: + pinned_version = pinned_versions.get(id(source)) + if source.get("connection_id"): + source_version = _snapshot_database_source( + db, svc, source, input_dataset_version_ids, + selected_version=pinned_version, + ) + else: + source_version = pinned_version + # SQL source identity is only known after its first snapshot. + # A multi-source pipeline cannot be represented by this one-source + # refresh binding; never silently bind it to the first input. + if not multi_source and not pl.source_dataset_id and source.get("dataset_id"): + pl.source_dataset_id = source["dataset_id"] + db.commit() source["route"] = _source_runtime_route(source, transform_route, pl.route) - source_version_ids = [] - for version_id in input_dataset_version_ids: - version = db.get(DatasetVersion, version_id) - if version is not None and version.dataset_id == source["dataset_id"]: - source_version_ids.append(version.id) - source_version = select_pinned_dataset_version( - db, - dataset_id=source["dataset_id"], - input_dataset_version_ids=source_version_ids or None, - ) + source_version_ids = [source_version.id] if pinned_version is not None else [] + if source_version is None: + source_version = select_pinned_dataset_version( + db, + dataset_id=source["dataset_id"], + input_dataset_version_ids=source_version_ids or None, + ) data = _load_source_rows( db, svc, source, input_dataset_version_ids=source_version_ids or None, @@ -544,17 +737,20 @@ def set_node_status(nid: str, status: str): "curated_dataset_id": curated_ids[0] if curated_ids else None, "curated_dataset_ids": curated_ids, } + _set_pipeline_status_if_latest(db, pl, run, "published") db.commit() except Exception as e: - logger.error(f"Pipeline run failed: {e}") + detail = safe_connection_error(e) + logger.error("Pipeline run failed: %s", detail) if run: run.status = "failed" - run.error_log = str(e) + run.error_log = detail run.finished_at = datetime.now(timezone.utc) stats = dict(run.stats or {}) stats.setdefault("node_status", {}) run.stats = stats + _set_pipeline_status_if_latest(db, pl, run, "failed") db.commit() finally: db.close() diff --git a/backend/app/tasks/v2/refresh_tasks.py b/backend/app/tasks/v2/refresh_tasks.py index b5b2138a..6bd262e6 100644 --- a/backend/app/tasks/v2/refresh_tasks.py +++ b/backend/app/tasks/v2/refresh_tasks.py @@ -27,12 +27,6 @@ logger = logging.getLogger(__name__) -# Placeholder resource scope for a connection/pipeline-level manual or -# scheduled refresh until multi-resource fan-out lands (matches -# schedule_service.DEFAULT_RESOURCE). -DEFAULT_RESOURCE = "__scheduled__" - - def _celery_send_task(task_name: str, args, queue: str) -> str: """Adapter matching `enqueue_refresh_run`'s `send_task` contract — Celery's own `send_task(name, args, kwargs, ...)` would otherwise bind a @@ -325,27 +319,28 @@ def create_manual_connection_run(db, connection_id: str): """ from app.models.v2.connection import Connection from app.schemas.refresh import RefreshError, RefreshPolicy - from app.services.v2.incremental.operations import _policy_for_source, _source_state + from app.services.v2.incremental.operations import ( + _policy_for_source, + _source_state, + resolve_refresh_resource, + ) from app.services.v2.incremental.contract import claim_refresh_run connection = db.get(Connection, connection_id) if connection is None: - # Preserve the historical internal helper behavior for callers that - # already provision source state without a Connection row. The - # authenticated route verifies the connection first; persisted - # policies are enforced whenever that source record exists. - policy = RefreshPolicy.BATCH - else: - configured_state = _source_state(db, source_id=connection_id) - policy = _policy_for_source(connection, configured_state) - if policy is RefreshPolicy.EVENT_DRIVEN: - raise RefreshError( - "EVENT_TRIGGER_REQUIRES_SIGNATURE", - "event-driven refreshes require the signed webhook route", - ) + raise RefreshError("SOURCE_NOT_FOUND", f"no source connection {connection_id}") + + resource = resolve_refresh_resource(db, source_id=connection_id) + configured_state = _source_state(db, source_id=connection_id, resource=resource) + policy = _policy_for_source(connection, configured_state) + if policy is RefreshPolicy.EVENT_DRIVEN: + raise RefreshError( + "EVENT_TRIGGER_REQUIRES_SIGNATURE", + "event-driven refreshes require the signed webhook route", + ) run = claim_refresh_run( - db, source_id=connection_id, resource=DEFAULT_RESOURCE, policy=policy, + db, source_id=connection_id, resource=resource, policy=policy, idempotency_key=f"manual:{uuid.uuid4()}", lease_owner="manual-sync", now=datetime.now(timezone.utc), ) diff --git a/backend/openapi-agent.json b/backend/openapi-agent.json index c43d7de1..f0bfbdb2 100644 --- a/backend/openapi-agent.json +++ b/backend/openapi-agent.json @@ -14617,6 +14617,119 @@ ] } }, + "/api/v2/connections/{connection_id}/resources": { + "get": { + "description": "List tables/collections exposed by a saved connection.", + "operationId": "list_connection_resources_api_v2_connections__connection_id__resources_get", + "parameters": [ + { + "in": "path", + "name": "connection_id", + "required": true, + "schema": { + "title": "Connection Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "HTTPBearer": [] + } + ], + "summary": "List Connection Resources", + "tags": [ + "v2-connections" + ] + } + }, + "/api/v2/connections/{connection_id}/resources/{resource}/sample": { + "get": { + "description": "Read a bounded sample from one saved connection resource.", + "operationId": "sample_connection_resource_api_v2_connections__connection_id__resources__resource__sample_get", + "parameters": [ + { + "in": "path", + "name": "connection_id", + "required": true, + "schema": { + "title": "Connection Id", + "type": "string" + } + }, + { + "in": "path", + "name": "resource", + "required": true, + "schema": { + "title": "Resource", + "type": "string" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "default": 20, + "maximum": 1000, + "minimum": 1, + "title": "Limit", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "HTTPBearer": [] + } + ], + "summary": "Sample Connection Resource", + "tags": [ + "v2-connections" + ] + } + }, "/api/v2/connections/{connection_id}/schedule": { "post": { "description": "Compatibility delegate for the typed refresh schedule operation.", diff --git a/backend/tests/v2/connection/test_connection_resources.py b/backend/tests/v2/connection/test_connection_resources.py new file mode 100644 index 00000000..5d7ce171 --- /dev/null +++ b/backend/tests/v2/connection/test_connection_resources.py @@ -0,0 +1,138 @@ +"""v2 saved database connection resource and sample API coverage.""" +import json +from unittest.mock import MagicMock, patch + +from app.models.v2.connection import Connection +from app.models.user import User +from app.routers.v2 import connections as connections_router +from app.services.auth_service import create_access_token, hash_password + + +def _override_connections_db(db): + def override(): + yield db + + return override + + +def test_saved_connection_keeps_credentials_encrypted_and_lists_resources( + client, db, auth_headers, admin_user, monkeypatch, +): + app = client.app + previous = app.dependency_overrides.get(connections_router.get_db) + app.dependency_overrides[connections_router.get_db] = _override_connections_db(db) + try: + created = client.post( + "/api/v2/connections", + headers=auth_headers, + json={ + "name": "runtime mysql", + "kind": "mysql", + "config": { + "host": "127.0.0.1", + "port": "53306", + "database": "runtime", + "user": "runtime", + "password": "secret", + }, + }, + ) + assert created.status_code == 201 + connection = db.get(Connection, created.json()["id"]) + assert connection is not None + assert "secret" not in json.dumps(connection.config) + assert "_encrypted" in connection.config + assert connection.created_by == admin_user.id + + fake = MagicMock() + fake.list_resources.return_value = ["codex_pipeline_probe"] + monkeypatch.setattr(connections_router, "get_connector", lambda kind, config: fake) + listed = client.get( + f"/api/v2/connections/{connection.id}/resources", + headers=auth_headers, + ) + + assert listed.status_code == 200 + assert listed.json()["resources"] == ["codex_pipeline_probe"] + fake.list_resources.assert_called_once_with() + finally: + if previous is None: + app.dependency_overrides.pop(connections_router.get_db, None) + else: + app.dependency_overrides[connections_router.get_db] = previous + + +def test_saved_connection_returns_sample_rows_for_a_table(client, db, auth_headers, monkeypatch): + app = client.app + previous = app.dependency_overrides.get(connections_router.get_db) + app.dependency_overrides[connections_router.get_db] = _override_connections_db(db) + try: + connection = Connection( + id="resource-sample-connection", + name="runtime postgres", + kind="postgres", + config={"_encrypted": "ignored-by-mock"}, + status="inactive", + ) + db.add(connection) + db.commit() + + fake = MagicMock() + fake.pull_sample.return_value = [{"id": 1, "customer": "Alice"}] + monkeypatch.setattr(connections_router, "_load_connection_connector", lambda *args, **kwargs: fake) + response = client.get( + f"/api/v2/connections/{connection.id}/resources/codex_pipeline_probe/sample?limit=1", + headers=auth_headers, + ) + + assert response.status_code == 200 + assert response.json()["rows"] == [{"id": 1, "customer": "Alice"}] + fake.pull_sample.assert_called_once_with("codex_pipeline_probe", limit=1) + finally: + if previous is None: + app.dependency_overrides.pop(connections_router.get_db, None) + else: + app.dependency_overrides[connections_router.get_db] = previous + + +def test_viewer_cannot_list_or_sample_saved_connection_resources(client, db): + app = client.app + previous = app.dependency_overrides.get(connections_router.get_db) + app.dependency_overrides[connections_router.get_db] = _override_connections_db(db) + try: + db.add(Connection( + id="viewer-resource-connection", + name="runtime postgres", + kind="postgres", + config={"_encrypted": "ignored-by-authz-test"}, + status="active", + )) + viewer = User( + id="resource-viewer", + username="resource-viewer", + email="resource-viewer@test.com", + password_hash=hash_password("viewer123"), + role="viewer", + ) + db.add(viewer) + db.commit() + headers = { + "Authorization": f"Bearer {create_access_token({'sub': viewer.id, 'role': viewer.role})}", + } + + listed = client.get( + "/api/v2/connections/viewer-resource-connection/resources", + headers=headers, + ) + sampled = client.get( + "/api/v2/connections/viewer-resource-connection/resources/orders/sample", + headers=headers, + ) + + assert listed.status_code == 403 + assert sampled.status_code == 403 + finally: + if previous is None: + app.dependency_overrides.pop(connections_router.get_db, None) + else: + app.dependency_overrides[connections_router.get_db] = previous diff --git a/backend/tests/v2/connection/test_scheduler.py b/backend/tests/v2/connection/test_scheduler.py index 1c086e89..287753e3 100644 --- a/backend/tests/v2/connection/test_scheduler.py +++ b/backend/tests/v2/connection/test_scheduler.py @@ -47,9 +47,8 @@ def test_schedule_connection_without_db_only_validates_never_reports_active(): def test_schedule_pipeline_without_db_only_validates_never_reports_active(): svc = CronService() - result = svc.schedule_pipeline_run("pl-1", "*/30 * * * *") - assert result["status"] != "scheduled" - assert result["pipeline_id"] == "pl-1" + with pytest.raises(ValueError, match="PIPELINE_SCHEDULE_UNSUPPORTED"): + svc.schedule_pipeline_run("pl-1", "*/30 * * * *") def test_schedule_connection_with_db_persists_and_returns_scheduled(db): @@ -69,10 +68,13 @@ def test_schedule_connection_with_db_persists_and_returns_scheduled(db): def test_schedule_pipeline_with_db_persists_and_returns_scheduled(db): svc = CronService() - result = svc.schedule_pipeline_run("pl-1", "*/30 * * * *", db=db) - assert result["status"] == "scheduled" - assert result["pipeline_id"] == "pl-1" - assert result["next_due_at"] is not None + with pytest.raises(ValueError, match="PIPELINE_SCHEDULE_UNSUPPORTED"): + svc.schedule_pipeline_run("pl-1", "*/30 * * * *", db=db) + + from app.models.v2.refresh import RefreshSchedule + assert db.query(RefreshSchedule).filter( + RefreshSchedule.target_type == "pipeline", RefreshSchedule.target_id == "pl-1", + ).count() == 0 def test_describe_cron_every_minute(): @@ -137,11 +139,11 @@ def scheduler_connection(db): db.add(Connection( id="scheduler-conn-001", name="scheduler-source", kind="rest", status="active", - config={"cursor_contract": "watermark_primary_key"}, + config={"cursor_contract": "watermark_primary_key", "resource": "orders"}, refresh_policy="micro_batch", cursor_contract="watermark_primary_key", )) db.add(RefreshSourceState( - id="scheduler-state-001", source_id="scheduler-conn-001", resource="__scheduled__", + id="scheduler-state-001", source_id="scheduler-conn-001", resource="orders", cursor_contract="watermark_primary_key", config_version=1, cursor_json={ "watermark": NOW.isoformat(), "primary_key": "1", "opaque_value": None, diff --git a/backend/tests/v2/connection/test_sql_connector.py b/backend/tests/v2/connection/test_sql_connector.py index 46acd331..79d34553 100644 --- a/backend/tests/v2/connection/test_sql_connector.py +++ b/backend/tests/v2/connection/test_sql_connector.py @@ -37,6 +37,49 @@ def test_registry_mysql_returns_sql_connector(): assert CONNECTOR_REGISTRY["postgres"] is SQLConnector +def test_saved_sql_connection_fields_are_compiled_to_a_safe_sqlalchemy_url(): + from app.services.connection.config import build_connector_config + + mysql = build_connector_config( + "mysql", + { + "host": "127.0.0.1", + "port": "3306", + "database": "runtime", + "user": "runtime", + "password": "p@ss:word/with#chars", + }, + ) + postgres = build_connector_config( + "postgres", + { + "host": "127.0.0.1", + "port": 5432, + "database": "runtime", + "user": "runtime", + "password": "secret", + }, + ) + + assert mysql["connection_string"] == ( + "mysql+pymysql://runtime:p%40ss%3Aword%2Fwith%23chars@127.0.0.1:3306/runtime" + ) + assert postgres["connection_string"] == "postgresql+psycopg2://runtime:secret@127.0.0.1:5432/runtime" + + +def test_sql_connector_keeps_actionable_failure_detail_without_exposing_password(): + from app.services.connection.sql_connector import SQLConnector + + connector = SQLConnector({ + "connection_string": "postgresql+psycopg2://user:secret@localhost:5432/runtime", + }) + connector._get_engine = MagicMock(side_effect=RuntimeError("connection refused")) + + assert connector.test_connection() is False + assert connector.last_error == "connection refused" + assert "secret" not in connector.last_error + + def test_get_connector_all_supported_kinds(): from app.services.connection.registry import get_connector for kind in ["file", "mysql", "postgres"]: diff --git a/backend/tests/v2/incremental/test_refresh_schedule.py b/backend/tests/v2/incremental/test_refresh_schedule.py index 9781dd4b..7933ecc3 100644 --- a/backend/tests/v2/incremental/test_refresh_schedule.py +++ b/backend/tests/v2/incremental/test_refresh_schedule.py @@ -24,9 +24,9 @@ import pytest from sqlalchemy import func, select +from app.models.v2.connection import Connection from app.models.v2.refresh import RefreshRun, RefreshSchedule from app.services.v2.scheduler.schedule_service import ( - DEFAULT_RESOURCE, ScheduleRequest, dispatch_due_schedules, upsert_refresh_schedule, @@ -51,6 +51,12 @@ def upsert_fixture_schedule(db, *, cron_expr, timezone, business_calendar, sla_s def create_fixture_schedule(db, *, target_id, max_pending_runs=5, cron_expr="* * * * *", retry_policy=None) -> RefreshSchedule: """A schedule already due at FIXED_NOW (every-minute cron, upserted one minute before FIXED_NOW so next_due_at lands exactly on it).""" + if db.get(Connection, target_id) is None: + db.add(Connection( + id=target_id, name=target_id, kind="rest", config={"resource": "orders"}, + status="active", refresh_policy="batch", cursor_contract="watermark_primary_key", + )) + db.commit() request = ScheduleRequest( target_type="connection", target_id=target_id, cron_expr=cron_expr, timezone="UTC", business_calendar=[], sla_seconds=0, retry_policy=retry_policy, backfill_window_seconds=0, @@ -59,7 +65,7 @@ def create_fixture_schedule(db, *, target_id, max_pending_runs=5, cron_expr="* * return upsert_refresh_schedule(db, request, now=FIXED_NOW - timedelta(minutes=1)) -def create_queued_fixture_run(db, *, source_id, resource=DEFAULT_RESOURCE) -> RefreshRun: +def create_queued_fixture_run(db, *, source_id, resource="orders") -> RefreshRun: """A pre-existing active run occupying the source's admission slot, independent of any schedule dispatch.""" run = RefreshRun( @@ -123,6 +129,12 @@ def test_scheduled_run_persists_destination_queue_for_operability(db): def test_manual_refresh_run_persists_destination_queue_for_operability(db): from app.tasks.v2.refresh_tasks import create_manual_connection_run + db.add(Connection( + id="source-manual-queue-attribution", name="manual source", kind="rest", + config={"resource": "orders"}, status="active", refresh_policy="batch", + cursor_contract="watermark_primary_key", + )) + db.commit() run = create_manual_connection_run(db, "source-manual-queue-attribution") assert run.dispatch_queue == "refresh.poll" @@ -287,7 +299,7 @@ def test_expired_dispatch_claim_is_recovered_once_without_duplicate(db): schedule = create_fixture_schedule(db, target_id="source-expired-claim") occurrence_key = f"schedule:{schedule.id}:{schedule.next_due_at.isoformat()}" retained = RefreshRun( - id=str(uuid.uuid4()), source_id="source-expired-claim", resource=DEFAULT_RESOURCE, + id=str(uuid.uuid4()), source_id="source-expired-claim", resource="orders", policy="batch", trigger="scheduled", config_version=1, cursor_contract="watermark_primary_key", status="queued", dispatch_state="pending", dispatch_queue="refresh.poll", dispatch_claim_owner="abandoned-beat", diff --git a/backend/tests/v2/incremental/test_resource_resolution.py b/backend/tests/v2/incremental/test_resource_resolution.py new file mode 100644 index 00000000..a6c223f8 --- /dev/null +++ b/backend/tests/v2/incremental/test_resource_resolution.py @@ -0,0 +1,195 @@ +"""Deterministic server-owned resource selection for refresh entry points.""" +from datetime import datetime, timedelta, timezone + +import pytest + +from app.models.v2.connection import Connection +from app.models.v2.dataset import Dataset +from app.models.v2.refresh import RefreshRun, RefreshSchedule, RefreshSourceState +from app.schemas.refresh import RefreshError, RefreshPolicy +from app.services.v2.incremental.operations import resolve_refresh_resource +from app.services.v2.scheduler.schedule_service import dispatch_due_schedules, upsert_refresh_schedule, ScheduleRequest +from app.tasks.v2.refresh_tasks import create_manual_connection_run + + +NOW = datetime(2026, 9, 24, tzinfo=timezone.utc) + + +def _connection(db, connection_id="resource-source", config=None): + connection = Connection( + id=connection_id, + name=connection_id, + kind="rest", + config=config or {}, + status="active", + refresh_policy=RefreshPolicy.BATCH.value, + cursor_contract="watermark_primary_key", + ) + db.add(connection) + db.commit() + return connection + + +def test_resource_config_wins_over_ambiguous_states(db): + _connection(db, config={"resource": "orders"}) + db.add_all([ + RefreshSourceState( + id="state-users", source_id="resource-source", resource="users", + cursor_contract="watermark_primary_key", config_version=1, + fencing_token=0, + ), + RefreshSourceState( + id="state-events", source_id="resource-source", resource="events", + cursor_contract="watermark_primary_key", config_version=1, + fencing_token=0, + ), + ]) + db.commit() + + assert resolve_refresh_resource(db, source_id="resource-source") == "orders" + + +def test_resource_resolution_ignores_legacy_sentinel_and_uses_one_real_state(db): + _connection(db) + db.add(RefreshSourceState( + id="state-sentinel", source_id="resource-source", resource="__scheduled__", + cursor_contract="watermark_primary_key", config_version=1, fencing_token=0, + )) + db.commit() + + with pytest.raises(RefreshError) as exc_info: + resolve_refresh_resource(db, source_id="resource-source") + assert exc_info.value.reason_code == "REFRESH_RESOURCE_REQUIRED" + + db.add(RefreshSourceState( + id="state-orders", source_id="resource-source", resource="orders", + cursor_contract="watermark_primary_key", config_version=1, fencing_token=0, + )) + db.commit() + assert resolve_refresh_resource(db, source_id="resource-source") == "orders" + + +def test_resource_resolution_uses_one_dataset_when_state_is_absent(db): + _connection(db) + db.add(Dataset( + id="dataset-orders", name="orders", source_connection_id="resource-source", + kind="structured", + )) + db.commit() + + assert resolve_refresh_resource(db, source_id="resource-source") == "orders" + + +def test_resource_resolution_rejects_ambiguous_state_candidates(db): + _connection(db) + db.add_all([ + RefreshSourceState( + id="state-orders", source_id="resource-source", resource="orders", + cursor_contract="watermark_primary_key", config_version=1, fencing_token=0, + ), + RefreshSourceState( + id="state-users", source_id="resource-source", resource="users", + cursor_contract="watermark_primary_key", config_version=1, fencing_token=0, + ), + ]) + db.commit() + + with pytest.raises(RefreshError) as exc_info: + resolve_refresh_resource(db, source_id="resource-source") + assert exc_info.value.reason_code == "REFRESH_RESOURCE_AMBIGUOUS" + + +def test_manual_compatibility_run_uses_concrete_resolved_resource(db): + _connection(db, config={"resource": "orders"}) + + run = create_manual_connection_run(db, "resource-source") + + assert run.resource == "orders" + assert db.query(RefreshRun).filter( + RefreshRun.source_id == "resource-source", RefreshRun.resource == "__scheduled__", + ).count() == 0 + + +def test_due_source_schedule_uses_concrete_resolved_resource(db): + _connection(db, config={"resource": "orders"}) + schedule = upsert_refresh_schedule( + db, + ScheduleRequest( + target_type="connection", target_id="resource-source", cron_expr="* * * * *", + timezone="UTC", business_calendar=[], sla_seconds=0, retry_policy=None, + backfill_window_seconds=0, max_pending_runs=1, enabled=True, + ), + now=NOW - timedelta(minutes=1), + ) + + sent = [] + dispatched = dispatch_due_schedules( + db, now=NOW, lease_owner="beat", send_refresh=lambda message: sent.append(message) or message.run_id, + ) + + run = db.query(RefreshRun).filter(RefreshRun.idempotency_key.like(f"schedule:{schedule.id}:%")).one() + assert dispatched == [run.id] + assert run.resource == "orders" + assert run.resource != "__scheduled__" + assert sent[0].task_name == "refresh.connection" + + +def test_pipeline_target_schedule_is_rejected_before_persistence(db): + from app.services.v2.scheduler.cron_service import CronService + + with pytest.raises(ValueError, match="PIPELINE_SCHEDULE_UNSUPPORTED"): + CronService().schedule_pipeline_run("pipeline-1", "*/30 * * * *", db=db) + + assert db.query(RefreshSchedule).filter( + RefreshSchedule.target_type == "pipeline", RefreshSchedule.target_id == "pipeline-1", + ).count() == 0 + + +def test_existing_pipeline_schedule_is_disabled_without_poll_run(db): + schedule = RefreshSchedule( + id="pipeline-schedule-1", target_type="pipeline", target_id="pipeline-1", + cron_expression="* * * * *", timezone="UTC", excluded_dates=[], + max_pending_runs=1, enabled=True, next_due_at=NOW, + ) + db.add(schedule) + db.commit() + + assert dispatch_due_schedules( + db, now=NOW, lease_owner="beat", send_refresh=lambda message: message.run_id, + ) == [] + db.refresh(schedule) + assert schedule.enabled is False + assert db.query(RefreshRun).filter(RefreshRun.source_id == "pipeline-1").count() == 0 + + +def test_bad_resource_schedule_does_not_block_good_schedule(db): + _connection(db, connection_id="good-resource-source", config={"resource": "orders"}) + _connection(db, connection_id="bad-resource-source") + good = upsert_refresh_schedule( + db, + ScheduleRequest( + target_type="connection", target_id="good-resource-source", cron_expr="* * * * *", + timezone="UTC", business_calendar=[], sla_seconds=0, retry_policy=None, + backfill_window_seconds=0, max_pending_runs=1, enabled=True, + ), + now=NOW - timedelta(minutes=1), + ) + bad = upsert_refresh_schedule( + db, + ScheduleRequest( + target_type="connection", target_id="bad-resource-source", cron_expr="* * * * *", + timezone="UTC", business_calendar=[], sla_seconds=0, retry_policy=None, + backfill_window_seconds=0, max_pending_runs=1, enabled=True, + ), + now=NOW - timedelta(minutes=1), + ) + + dispatched = dispatch_due_schedules( + db, now=NOW, lease_owner="beat", send_refresh=lambda message: message.run_id, + ) + + assert len(dispatched) == 1 + run = db.query(RefreshRun).filter(RefreshRun.idempotency_key.like(f"schedule:{good.id}:%")).one() + assert dispatched == [run.id] + db.refresh(bad) + assert bad.next_due_at.replace(tzinfo=timezone.utc) > NOW diff --git a/backend/tests/v2/pipeline/test_database_source.py b/backend/tests/v2/pipeline/test_database_source.py new file mode 100644 index 00000000..7d5925ad --- /dev/null +++ b/backend/tests/v2/pipeline/test_database_source.py @@ -0,0 +1,126 @@ +"""Database connector source -> snapshot -> Route A pipeline coverage.""" +from unittest.mock import MagicMock + +from sqlalchemy.orm import sessionmaker + +from app.models.v2.connection import Connection +from app.models.v2.pipeline import Pipeline, PipelineRun, PipelineRunInput +from app.services import encryption_service +from app.tasks.v2 import pipeline_run as pipeline_module +from app.routers.v2.pipelines import _sanitize_definition + + +def test_pipeline_definition_strips_database_credentials_before_persisting(): + definition = { + "nodes": [{ + "type": "connector", + "config": { + "source_type": "postgresql", + "connection_id": "db-source-001", + "resource": "codex_pipeline_probe", + "config_values": { + "host": "127.0.0.1", + "password": "do-not-store", + }, + "password": "do-not-store-either", + "credentials": { + "username": "runtime", + "password": "nested-do-not-store", + }, + }, + }], + "edges": [], + } + + sanitized = _sanitize_definition(definition) + + assert sanitized is not definition + config = sanitized["nodes"][0]["config"] + assert config["connection_id"] == "db-source-001" + assert "password" not in config + assert "config_values" not in config + assert "credentials" not in config + assert definition["nodes"][0]["config"]["password"] == "do-not-store-either" + + +def test_pipeline_run_snapshots_selected_database_table_and_records_lineage(db, monkeypatch): + class MemoryStorage: + def __init__(self): + self.objects = {} + + def put_bytes(self, bucket, key, data, content_type="application/octet-stream"): + uri = f"s3://{bucket}/{key}" + self.objects[uri] = data + return uri + + def get_object(self, uri): + return self.objects[uri] + + storage = MemoryStorage() + monkeypatch.setattr("app.database.SessionLocal", sessionmaker(bind=db.get_bind())) + monkeypatch.setattr("app.services.v2.dataset_service.get_storage_service", lambda: storage) + connector = MagicMock() + connector.pull_full.return_value = [ + {"id": row_id, "customer": f"Customer {row_id}", "amount": row_id} + for row_id in range(10001) + ] + monkeypatch.setattr(pipeline_module, "get_connector", lambda kind, config: connector) + monkeypatch.setattr( + pipeline_module, + "_execute_route", + lambda route, context, data: (data, context), + ) + + connection = Connection( + id="db-source-001", + name="runtime postgres", + kind="postgres", + config={"_encrypted": encryption_service.encrypt( + '{"host":"127.0.0.1","port":"55432","database":"runtime",' + '"user":"runtime","password":"secret"}' + )}, + status="active", + ) + pipeline = Pipeline( + id="db-pipeline-001", + name="database pipeline", + route="A", + spec={}, + definition={ + "nodes": [{ + "id": "source", + "type": "connector", + "config": { + "source_type": "postgresql", + "connection_id": connection.id, + "resource": "codex_pipeline_probe", + }, + }], + "edges": [], + }, + status="draft", + ) + run = PipelineRun(id="db-run-001", pipeline_id=pipeline.id, status="pending") + db.add_all([connection, pipeline, run]) + db.commit() + + pipeline_module.pipeline_run_task.run(pipeline.id, run.id) + db.expire_all() + + persisted = db.get(PipelineRun, run.id) + assert persisted.status == "success" + assert persisted.dataset_version_id is not None + assert persisted.stats["rows_in"] == 10001 + assert db.query(PipelineRunInput).filter( + PipelineRunInput.pipeline_run_id == run.id, + ).count() >= 2 + connector.pull_full.assert_called_once_with("codex_pipeline_probe") + from app.models.v2.dataset import Dataset + from app.services.v2.incremental.polling import _resolve_pipeline + + source_dataset = db.query(Dataset).filter( + Dataset.source_connection_id == connection.id, + Dataset.name == "codex_pipeline_probe", + ).one() + assert db.get(Pipeline, pipeline.id).source_dataset_id == source_dataset.id + assert _resolve_pipeline(db, dataset=source_dataset).id == pipeline.id diff --git a/backend/tests/v2/pipeline/test_pinned_inputs.py b/backend/tests/v2/pipeline/test_pinned_inputs.py new file mode 100644 index 00000000..b24d9c47 --- /dev/null +++ b/backend/tests/v2/pipeline/test_pinned_inputs.py @@ -0,0 +1,104 @@ +"""Pipeline input pinning and pipeline-level status ordering regressions.""" +from datetime import datetime, timezone +from sqlalchemy.orm import sessionmaker + +from app.models.v2.dataset import Dataset, DatasetVersion +from app.models.v2.pipeline import Pipeline, PipelineRun +from app.tasks.v2 import pipeline_run as pipeline_module + + +def _file_pipeline(db, *, pipeline_id="pinned-pipeline"): + first = Dataset(id=f"{pipeline_id}-dataset-a", name="a", kind="structured") + second = Dataset(id=f"{pipeline_id}-dataset-b", name="b", kind="structured") + first_version = DatasetVersion( + id=f"{pipeline_id}-version-a", dataset_id=first.id, version_no=1, rowcount=1, + ) + second_version = DatasetVersion( + id=f"{pipeline_id}-version-b", dataset_id=second.id, version_no=1, rowcount=1, + ) + pipeline = Pipeline( + id=pipeline_id, name=pipeline_id, spec={}, status="draft", + definition={ + "nodes": [{ + "id": "source-a", "type": "connector", + "config": {"source_type": "file", "files": [{"dataset_id": first.id, "name": "a.csv"}]}, + }, { + "id": "source-b", "type": "connector", + "config": {"source_type": "file", "files": [{"dataset_id": second.id, "name": "b.csv"}]}, + }], + "edges": [], + }, + ) + db.add_all([first, second, first_version, second_version, pipeline]) + db.commit() + return pipeline, first_version, second_version + + +def test_incomplete_pinned_inputs_fail_before_latest_lookup(db, monkeypatch): + pipeline, first_version, _ = _file_pipeline(db) + run = PipelineRun(id="pinned-run-incomplete", pipeline_id=pipeline.id, status="pending") + db.add(run) + db.commit() + + def fail_if_latest_lookup(*args, **kwargs): + raise AssertionError("latest version lookup must not run for incomplete pins") + + monkeypatch.setattr( + "app.services.v2.incremental.polling.select_pinned_dataset_version", + fail_if_latest_lookup, + ) + monkeypatch.setattr("app.database.SessionLocal", sessionmaker(bind=db.get_bind())) + pipeline_module.pipeline_run_task.run(pipeline.id, run.id, [first_version.id]) + db.expire_all() + + persisted = db.get(PipelineRun, run.id) + assert persisted.status == "failed" + assert persisted.error_log.startswith("PINNED_INPUTS_INCOMPLETE") + + +def test_duplicate_pins_for_one_source_are_rejected(db, monkeypatch): + pipeline, first_version, _ = _file_pipeline(db, pipeline_id="pinned-pipeline-duplicate") + replacement = DatasetVersion( + id="pinned-pipeline-duplicate-version-a-2", + dataset_id=first_version.dataset_id, + version_no=2, + rowcount=1, + ) + db.add(replacement) + run = PipelineRun(id="pinned-run-duplicate", pipeline_id=pipeline.id, status="pending") + db.add(run) + db.commit() + + monkeypatch.setattr("app.database.SessionLocal", sessionmaker(bind=db.get_bind())) + pipeline_module.pipeline_run_task.run( + pipeline.id, run.id, [first_version.id, replacement.id], + ) + db.expire_all() + + persisted = db.get(PipelineRun, run.id) + assert persisted.status == "failed" + assert persisted.error_log.startswith("PINNED_INPUTS_INCOMPLETE") + + +def test_pipeline_status_summary_only_follows_latest_run(db): + from app.tasks.v2.pipeline_run import _set_pipeline_status_if_latest + + pipeline = Pipeline(id="status-pipeline", name="status-pipeline", spec={}, status="draft") + older = PipelineRun( + id="status-run-a", pipeline_id=pipeline.id, status="failed", + finished_at=datetime(2026, 9, 24, 0, 0, tzinfo=timezone.utc), + error_log="older failure", + created_at=datetime(2026, 9, 24, 0, 0, tzinfo=timezone.utc), + ) + newer = PipelineRun( + id="status-run-b", pipeline_id=pipeline.id, status="running", + created_at=datetime(2026, 9, 24, 0, 1, tzinfo=timezone.utc), + ) + db.add_all([pipeline, older, newer]) + db.commit() + + assert _set_pipeline_status_if_latest(db, pipeline, older, "published") is False + assert db.get(Pipeline, pipeline.id).status == "draft" + + assert _set_pipeline_status_if_latest(db, pipeline, newer, "published") is True + assert db.get(Pipeline, pipeline.id).status == "published" diff --git a/backend/tests/v2/test_pipeline_connector_safety.py b/backend/tests/v2/test_pipeline_connector_safety.py new file mode 100644 index 00000000..c5180ce3 --- /dev/null +++ b/backend/tests/v2/test_pipeline_connector_safety.py @@ -0,0 +1,195 @@ +"""Builder pipeline connector safety contracts.""" + +import uuid + +import pytest + +from app.models.v2.pipeline import Pipeline + + +@pytest.fixture +def pipelines_client(client, db): + """Use the test session for the v2 pipeline router.""" + from app.main import app + from app.routers.v2 import pipelines + + def _override(): + yield db + + app.dependency_overrides[pipelines.get_db] = _override + yield client + app.dependency_overrides.pop(pipelines.get_db, None) + + +def _headers(client, username="admin", password="admin123"): + response = client.post( + "/api/v1/auth/login", + json={"username": username, "password": password}, + ) + return {"Authorization": f"Bearer {response.json()['data']['access_token']}"} + + +def _inline_definition(source_type: str) -> dict: + return { + "nodes": [{ + "id": "source", + "type": "connector", + "config": { + "source_type": source_type, + "config_values": { + "uri": "mongodb://user:secret@example.invalid/orders" + if source_type == "mongodb" + else "https://example.invalid/orders", + "headers": '{"Authorization":"Bearer secret"}', + }, + }, + }], + "edges": [], + } + + +@pytest.mark.parametrize("source_type", ["mongodb", "rest_api"]) +def test_create_rejects_unsupported_inline_connectors_before_persistence( + pipelines_client, db, admin_user, source_type, +): + name = f"unsupported-{source_type}-{uuid.uuid4().hex}" + + response = pipelines_client.post( + "/api/v2/pipelines", + json={"name": name, "definition": _inline_definition(source_type)}, + headers=_headers(pipelines_client), + ) + + assert response.status_code == 422 + assert response.json()["detail"] == "UNSUPPORTED_INLINE_CONNECTOR" + assert db.query(Pipeline).filter(Pipeline.name == name).count() == 0 + + +def test_update_rejects_unsupported_inline_connector_before_mutating_definition( + pipelines_client, db, admin_user, +): + pipeline = Pipeline( + name=f"existing-{uuid.uuid4().hex}", + route="A", + spec={}, + definition={"nodes": [], "edges": []}, + ) + db.add(pipeline) + db.commit() + pipeline_id = pipeline.id + + response = pipelines_client.put( + f"/api/v2/pipelines/{pipeline_id}", + json={"definition": _inline_definition("rest_api")}, + headers=_headers(pipelines_client), + ) + + assert response.status_code == 422 + assert response.json()["detail"] == "UNSUPPORTED_INLINE_CONNECTOR" + assert db.get(Pipeline, pipeline_id).definition == {"nodes": [], "edges": []} + + +def test_validate_marks_legacy_unsupported_connector_definition_invalid( + pipelines_client, db, admin_user, +): + pipeline = Pipeline( + name=f"legacy-{uuid.uuid4().hex}", + route="A", + spec={}, + definition=_inline_definition("mongodb"), + ) + db.add(pipeline) + db.commit() + + response = pipelines_client.post( + f"/api/v2/pipelines/{pipeline.id}/validate", + headers=_headers(pipelines_client), + ) + + assert response.status_code == 200 + payload = response.json() + assert payload["valid"] is False + assert payload["errors"] == [{ + "node_id": "source", + "severity": "error", + "code": "UNSUPPORTED_INLINE_CONNECTOR", + "message": "MongoDB/REST inline connector nodes are unsupported in Builder; use File Upload in Builder and manage external connections in Data Connections.", + }] + + +def test_legacy_inline_connector_secrets_are_not_returned_by_pipeline_detail( + pipelines_client, db, admin_user, +): + pipeline = Pipeline( + name=f"legacy-secret-{uuid.uuid4().hex}", + route="A", + spec={}, + definition=_inline_definition("rest_api"), + ) + db.add(pipeline) + db.commit() + + response = pipelines_client.get( + f"/api/v2/pipelines/{pipeline.id}", + headers=_headers(pipelines_client), + ) + + assert response.status_code == 200 + assert response.json()["definition"]["nodes"][0]["config"] == {"source_type": "rest_api"} + assert "Bearer secret" not in response.text + + +def test_legacy_inline_connector_cannot_queue_a_run(pipelines_client, db, admin_user): + pipeline = Pipeline( + name=f"legacy-run-{uuid.uuid4().hex}", + route="A", + spec={}, + definition=_inline_definition("mongodb"), + status="draft", + ) + db.add(pipeline) + db.commit() + + response = pipelines_client.post( + f"/api/v2/pipelines/{pipeline.id}/run", + headers=_headers(pipelines_client), + ) + + assert response.status_code == 422 + assert response.json()["detail"] == "UNSUPPORTED_INLINE_CONNECTOR" + db.refresh(pipeline) + assert pipeline.status == "draft" + + +def test_saved_sql_source_does_not_store_legacy_inline_credentials( + pipelines_client, db, admin_user, +): + definition = { + "nodes": [{ + "id": "sql-source", "type": "connector", + "config": { + "source_type": "postgresql", + "connection_id": "saved-connection", + "resource": "orders", + "config_values": { + "uri": "postgresql://user:secret@example.invalid/orders", + "headers": {"Authorization": "Bearer secret"}, + }, + }, + }], + "edges": [], + } + response = pipelines_client.post( + "/api/v2/pipelines", + json={"name": f"safe-sql-{uuid.uuid4().hex}", "definition": definition}, + headers=_headers(pipelines_client), + ) + + assert response.status_code == 201 + stored = db.get(Pipeline, response.json()["id"]).definition + assert stored["nodes"][0]["config"] == { + "source_type": "postgresql", + "connection_id": "saved-connection", + "resource": "orders", + } + assert "secret" not in response.text diff --git a/frontend/src/i18n/en.json b/frontend/src/i18n/en.json index 059c55f6..50ee752c 100644 --- a/frontend/src/i18n/en.json +++ b/frontend/src/i18n/en.json @@ -1308,7 +1308,15 @@ "test_failed": "Failed", "test_testing": "Testing...", "test_connection": "Test Connection", - "sync_mode_label": "Sync Mode" + "sync_mode_label": "Sync Mode", + "saved_connection_label": "Saved Connection", + "source_table_label": "Source Table", + "select_connection": "Select a saved connection", + "select_table": "Select a table", + "loading_connections": "Loading connections...", + "loading_resources": "Loading tables...", + "no_saved_connections": "Create a database connection in Data Connections first", + "resources_failed": "Could not discover tables" }, "outputInspector": { "output_type_label": "Output Type", @@ -1413,7 +1421,16 @@ "drop_release": "Release to add files", "drop_hint": "Drag files here, or", "drop_hint_click": "click to select", - "drop_formats_hint": "Supports CSV, XLSX, JSON, PDF, DOCX and more, multiple selection allowed" + "drop_formats_hint": "Supports CSV, XLSX, JSON, PDF, DOCX and more, multiple selection allowed", + "test_connection": "Test Connection", + "testing": "Testing...", + "test_success": "Connection succeeded", + "test_failed": "Connection failed", + "discover_tables": "Discover Tables", + "hide_tables": "Hide Tables", + "no_tables": "No tables found", + "resources_failed": "Could not discover tables", + "pipeline_sync_hint": "Run a selected table from Pipeline Builder" }, "curatedReview": { "notes_ph": "Review notes (optional)", diff --git a/frontend/src/i18n/zh.json b/frontend/src/i18n/zh.json index d9b50b05..48905891 100644 --- a/frontend/src/i18n/zh.json +++ b/frontend/src/i18n/zh.json @@ -1308,7 +1308,15 @@ "test_failed": "失败", "test_testing": "测试中...", "test_connection": "测试连接", - "sync_mode_label": "同步模式" + "sync_mode_label": "同步模式", + "saved_connection_label": "已保存连接", + "source_table_label": "源数据表", + "select_connection": "选择已保存的连接", + "select_table": "选择数据表", + "loading_connections": "正在加载连接...", + "loading_resources": "正在加载数据表...", + "no_saved_connections": "请先在数据连接中创建数据库连接", + "resources_failed": "无法发现数据表" }, "outputInspector": { "output_type_label": "输出类型", @@ -1413,7 +1421,16 @@ "drop_release": "松开以添加文件", "drop_hint": "拖拽文件到此处,或", "drop_hint_click": "点击选择", - "drop_formats_hint": "支持 CSV、XLSX、JSON、PDF、DOCX 等格式,可多选" + "drop_formats_hint": "支持 CSV、XLSX、JSON、PDF、DOCX 等格式,可多选", + "test_connection": "测试连接", + "testing": "测试中...", + "test_success": "连接成功", + "test_failed": "连接失败", + "discover_tables": "发现数据表", + "hide_tables": "收起数据表", + "no_tables": "未发现数据表", + "resources_failed": "无法发现数据表", + "pipeline_sync_hint": "请在 Pipeline Builder 中选择数据表并运行" }, "curatedReview": { "notes_ph": "审核备注(可选)", diff --git a/frontend/src/pages/pipelines/builder/inspectors/ConnectorInspector.test.tsx b/frontend/src/pages/pipelines/builder/inspectors/ConnectorInspector.test.tsx index af214f6a..b3cac076 100644 --- a/frontend/src/pages/pipelines/builder/inspectors/ConnectorInspector.test.tsx +++ b/frontend/src/pages/pipelines/builder/inspectors/ConnectorInspector.test.tsx @@ -1,15 +1,40 @@ import '@/i18n' -import { describe, expect, it, vi } from 'vitest' -import { render, screen, fireEvent, waitFor } from '@testing-library/react' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { act, render, screen, fireEvent, waitFor } from '@testing-library/react' +import { useState } from 'react' import { apiClientV2 } from '@/api/client' import ConnectorInspector from './ConnectorInspector' vi.mock('@/api/client', () => ({ - apiClientV2: { post: vi.fn() }, + apiClientV2: { post: vi.fn(), get: vi.fn() }, })) describe('ConnectorInspector test-connection button', () => { - const dbConfig = { source_type: 'postgresql', config_values: { host: 'localhost', port: '5432' } } + const dbConfig = { source_type: 'postgresql', connection_id: 'conn-1', config_values: { host: 'localhost', port: '5432' } } + + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(apiClientV2.get).mockResolvedValue([]) + }) + + it('never displays tables from a previously selected connection', async () => { + let finishSecond!: (value: { resources: string[] }) => void + const second = new Promise<{ resources: string[] }>(resolve => { finishSecond = resolve }) + vi.mocked(apiClientV2.get).mockImplementation(url => { + if (url === '/connections/a/resources') return Promise.resolve({ resources: ['a_table'] }) as never + if (url === '/connections/b/resources') return second as never + return Promise.resolve([]) as never + }) + const view = render( {}} />) + await screen.findByRole('option', { name: 'a_table' }) + view.rerender( {}} />) + expect(screen.queryByRole('option', { name: 'a_table' })).toBeNull() + expect(screen.getByLabelText(/源数据表|source table/i)).toBeDisabled() + view.rerender( {}} />) + await act(async () => { finishSecond({ resources: ['b_table'] }); await second }) + expect(screen.queryByRole('option', { name: 'b_table' })).toBeNull() + expect(screen.getByLabelText(/源数据表|source table/i)).toBeDisabled() + }) it('shows failed when the backend reports success: false (e.g. wrong credentials) instead of treating any 200 response as success', async () => { vi.mocked(apiClientV2.post).mockResolvedValueOnce({ success: false, detail: 'connection refused' }) @@ -32,4 +57,85 @@ describe('ConnectorInspector test-connection button', () => { expect(screen.getByRole('button', { name: /连接成功|success/i })).toBeTruthy() }) }) + + it('does not test SQL with inline config when no saved connection is selected', async () => { + render( {}} />) + + fireEvent.click(screen.getByRole('button', { name: /测试连接|Test Connection/i })) + + await waitFor(() => expect(screen.getByRole('button', { name: /select a saved connection|选择已保存的连接/i })).toBeTruthy()) + expect(apiClientV2.post).not.toHaveBeenCalled() + }) + + it('keeps file sources as a local readiness check', async () => { + render( {}} />) + + fireEvent.click(screen.getByRole('button', { name: /测试连接|Test Connection/i })) + + await waitFor(() => expect(screen.getByRole('button', { name: /连接成功|success/i })).toBeTruthy()) + expect(apiClientV2.post).not.toHaveBeenCalledWith('/connections/test-config', expect.anything()) + }) + + it('selects a saved database connection and one of its discovered tables', async () => { + vi.mocked(apiClientV2.get).mockImplementation(async (url: string) => { + if (url === '/connections') return [{ id: 'conn-1', name: 'Runtime PostgreSQL', kind: 'postgres', status: 'active' }] + if (url === '/connections/conn-1/resources') return { resources: ['codex_pipeline_probe'] } + return [] + }) + const onChange = vi.fn() + function Harness() { + const [config, setConfig] = useState>({ source_type: 'postgresql' }) + return { onChange(key, value); setConfig(previous => ({ ...previous, [key]: value })) }} /> + } + render() + + const savedConnection = await screen.findByLabelText(/已保存连接|saved connection/i) + fireEvent.change(savedConnection, { target: { value: 'conn-1' } }) + const table = await screen.findByLabelText(/源数据表|source table/i) + await screen.findByRole('option', { name: 'codex_pipeline_probe' }) + fireEvent.change(table, { target: { value: 'codex_pipeline_probe' } }) + + expect(onChange).toHaveBeenCalledWith('connection_id', 'conn-1') + expect(onChange).toHaveBeenCalledWith('resource', 'codex_pipeline_probe') + expect(onChange).not.toHaveBeenCalledWith('config_values', expect.objectContaining({ password: expect.anything() })) + }) + + it('does not offer unsupported MongoDB or REST source types in Builder', () => { + render( {}} />) + + expect(screen.queryByRole('option', { name: /MongoDB/i })).toBeNull() + expect(screen.queryByRole('option', { name: /REST API/i })).toBeNull() + }) + + it('renders legacy MongoDB and REST nodes as unsupported without exposing inline secrets', () => { + const { rerender } = render( {}} + readOnly + />) + + expect(screen.getByRole('alert')).toHaveTextContent(/unsupported in Builder/i) + expect(screen.queryByText(/mongodb:\/\/user:secret/i)).toBeNull() + expect(screen.queryByText(/secret/i)).toBeNull() + expect(screen.queryByRole('textbox')).toBeNull() + + rerender( {}} + readOnly + />) + + expect(screen.getByRole('alert')).toHaveTextContent(/unsupported in Builder/i) + expect(screen.queryByText(/Authorization|Bearer secret|example\.invalid/i)).toBeNull() + expect(screen.queryByRole('textbox')).toBeNull() + }) }) diff --git a/frontend/src/pages/pipelines/builder/inspectors/ConnectorInspector.tsx b/frontend/src/pages/pipelines/builder/inspectors/ConnectorInspector.tsx index b6d86c97..61ae8ff6 100644 --- a/frontend/src/pages/pipelines/builder/inspectors/ConnectorInspector.tsx +++ b/frontend/src/pages/pipelines/builder/inspectors/ConnectorInspector.tsx @@ -1,17 +1,31 @@ -import { useState, useCallback } from 'react' +import { useState, useCallback, useEffect } from 'react' import { useTranslation } from 'react-i18next' import { useDropzone } from 'react-dropzone' import { Upload, X, FileUp, Loader2, CheckCircle, XCircle } from 'lucide-react' import { apiClientV2 } from '@/api/client' -const SOURCE_LABEL: Record = { postgresql: 'PostgreSQL', mysql: 'MySQL', mongodb: 'MongoDB', rest_api: 'REST API' } +const SOURCE_LABEL: Record = { + postgresql: 'PostgreSQL', postgres: 'PostgreSQL', mysql: 'MySQL', mongo: 'MongoDB', mongodb: 'MongoDB', rest: 'REST API', rest_api: 'REST API', +} +const SQL_SOURCE_TYPES = new Set(['postgresql', 'postgres', 'mysql']) +const UNSUPPORTED_SOURCE_TYPES = new Set(['mongo', 'mongodb', 'rest', 'rest_api']) const DB_CONFIG_FIELDS: Record = { + // Retained for existing unsaved builder definitions. New pipeline database + // nodes use a saved connection selector below and never persist these fields. postgresql: [{ key: 'host', labelKey: 'connectorInspector.field_host', placeholder: 'localhost' }, { key: 'port', labelKey: 'connectorInspector.field_port', placeholder: '5432' }, { key: 'database', labelKey: 'connectorInspector.field_database', placeholder: 'mydb' }, { key: 'user', labelKey: 'connectorInspector.field_user', placeholder: 'postgres' }, { key: 'password', labelKey: 'connectorInspector.field_password', placeholder: '••••••', type: 'password' }], + postgres: [{ key: 'host', labelKey: 'connectorInspector.field_host', placeholder: 'localhost' }, { key: 'port', labelKey: 'connectorInspector.field_port', placeholder: '5432' }, { key: 'database', labelKey: 'connectorInspector.field_database', placeholder: 'mydb' }, { key: 'user', labelKey: 'connectorInspector.field_user', placeholder: 'postgres' }, { key: 'password', labelKey: 'connectorInspector.field_password', placeholder: '••••••', type: 'password' }], mysql: [{ key: 'host', labelKey: 'connectorInspector.field_host', placeholder: 'localhost' }, { key: 'port', labelKey: 'connectorInspector.field_port', placeholder: '3306' }, { key: 'database', labelKey: 'connectorInspector.field_database', placeholder: 'mydb' }, { key: 'user', labelKey: 'connectorInspector.field_user', placeholder: 'root' }, { key: 'password', labelKey: 'connectorInspector.field_password', placeholder: '••••••', type: 'password' }], mongodb: [{ key: 'uri', labelKey: 'connectorInspector.field_uri', placeholder: 'mongodb://localhost:27017/mydb' }], rest_api: [{ key: 'url', labelKey: '', label: 'API URL', placeholder: 'https://api.example.com/data' }, { key: 'headers', labelKey: 'connectorInspector.field_headers', placeholder: '{"Authorization":"Bearer token"}' }, { key: 'method', labelKey: 'connectorInspector.field_method', placeholder: 'GET' }], } +interface SavedConnection { + id: string + name: string + kind: string + status: string +} + interface UploadedFileMeta { name: string size: number @@ -21,15 +35,67 @@ interface UploadedFileMeta { const NO_FILES: UploadedFileMeta[] = [] +function asList(value: unknown, key: string): T[] { + if (Array.isArray(value)) return value as T[] + if (value && typeof value === 'object' && Array.isArray((value as Record)[key])) { + return (value as Record)[key] as T[] + } + return [] +} + export default function ConnectorInspector({ config, onChange, readOnly = false }: { config: Record; onChange: (key: string, value: unknown) => void; readOnly?: boolean }) { const { t } = useTranslation() const sourceType = String(config.source_type || 'file') + const isSqlSource = SQL_SOURCE_TYPES.has(sourceType) + const isUnsupportedSource = UNSUPPORTED_SOURCE_TYPES.has(sourceType) const cv = (config.config_values || {}) as Record const storedFiles = (config.files || NO_FILES) as UploadedFileMeta[] + const connectionId = String(config.connection_id || '') + const resource = String(config.resource || '') + const [connections, setConnections] = useState([]) + const resourceKey = `${sourceType}:${connectionId}` + const [resourceResult, setResourceResult] = useState<{ key: string; items: string[]; error: string } | null>(null) + const currentResources = resourceResult?.key === resourceKey ? resourceResult : null + const resources = currentResources?.items ?? [] + const resourceError = currentResources?.error ?? '' + const resourceLoading = Boolean(isSqlSource && connectionId && !currentResources) + const [connectionLoading, setConnectionLoading] = useState(false) const [uploading, setUploading] = useState(false) const [uploadError, setUploadError] = useState('') const [testStatus, setTestStatus] = useState<'idle' | 'testing' | 'success' | 'failed'>('idle') const [testMessage, setTestMessage] = useState('') + + useEffect(() => { + if (!isSqlSource) { + return + } + let active = true + void Promise.resolve() + .then(() => { + if (active) setConnectionLoading(true) + return apiClientV2.get('/connections') + }) + .then(value => { if (active) setConnections(asList(value, 'connections')) }) + .catch(() => { if (active) setConnections([]) }) + .finally(() => { if (active) setConnectionLoading(false) }) + return () => { active = false } + }, [isSqlSource]) + + useEffect(() => { + if (!isSqlSource || !connectionId) { + return + } + let active = true + void apiClientV2.get<{ resources?: string[] }>(`/connections/${connectionId}/resources`) + .then(value => { if (active) setResourceResult({ key: resourceKey, items: asList(value, 'resources'), error: '' }) }) + .catch((e: unknown) => { + if (!active) return + const err = e as { detail?: string; message?: string } | null | undefined + setResourceResult({ key: resourceKey, items: [], error: err?.detail || err?.message || t('connectorInspector.resources_failed') }) + }) + return () => { active = false } + }, [connectionId, isSqlSource, resourceKey, t]) + const onDrop = useCallback(async (accepted: File[]) => { if (accepted.length === 0) return setUploading(true) @@ -56,9 +122,21 @@ export default function ConnectorInspector({ config, onChange, readOnly = false const { getRootProps, getInputProps, isDragActive, open } = useDropzone({ onDrop, multiple: true, noClick: true }) const hasStoredFiles = storedFiles.length > 0 - const hasDbConfig = sourceType !== 'file' && Object.keys(cv).length > 0 - + const hasDbConfig = !isUnsupportedSource && sourceType !== 'file' && (Boolean(connectionId && resource) || Object.keys(cv).length > 0) const formatSize = (s: number | undefined) => s ? `(${(s / 1024).toFixed(1)} KB)` : '' + const selectedKind = sourceType === 'postgresql' ? 'postgres' : sourceType + const matchingConnections = connections.filter(connection => connection.kind === selectedKind || (selectedKind === 'postgres' && connection.kind === 'postgresql')) + + if (isUnsupportedSource) { + return ( +
+
+

{SOURCE_LABEL[sourceType] || sourceType}

+

{t('connectorInspector.unsupported_source', 'This source is unsupported in Builder. Use File Upload in Builder and manage external connections in Data Connections.')}

+
+
+ ) + } if (readOnly) { return ( @@ -69,7 +147,9 @@ export default function ConnectorInspector({ config, onChange, readOnly = false {sourceType === 'file' && hasStoredFiles && storedFiles.map((f, i: number) => (

📄 {f.name} {formatSize(f.size)}

))} - {sourceType !== 'file' && hasDbConfig && Object.entries(cv).filter(([k]) => k !== 'password').map(([k, v]) => ( + {isSqlSource && connectionId &&

{t('connectorInspector.saved_connection_label')}: {connectionId}

} + {isSqlSource && resource &&

{t('connectorInspector.source_table_label')}: {resource}

} + {sourceType !== 'file' && !isSqlSource && hasDbConfig && Object.entries(cv).filter(([k]) => k !== 'password').map(([k, v]) => (

{k}: {String(v).slice(0, 30)}

))} {!hasStoredFiles && !hasDbConfig &&

{t('connectorInspector.no_config')}

} @@ -78,10 +158,57 @@ export default function ConnectorInspector({ config, onChange, readOnly = false ) } + const handleSourceTypeChange = (value: string) => { + onChange('source_type', value) + onChange('connection_id', '') + onChange('resource', '') + onChange('config_values', {}) + onChange('files', []) + setTestStatus('idle') + } + + const handleConnectionChange = (value: string) => { + onChange('connection_id', value) + onChange('resource', '') + // A saved connection is the only runtime credential source. This also + // clears any legacy password entered in an older unsaved node. + onChange('config_values', {}) + setTestStatus('idle') + } + + const testConnection = async () => { + if (sourceType === 'file') { + setTestStatus(hasStoredFiles ? 'success' : 'failed') + setTestMessage(hasStoredFiles ? t('connectorInspector.test_ready') : t('connectorInspector.test_please_upload')) + return + } + if (isSqlSource && !connectionId) { + setTestStatus('failed') + setTestMessage(t('connectorInspector.select_connection')) + return + } + setTestStatus('testing') + try { + const result = isSqlSource && connectionId + ? await apiClientV2.post(`/connections/${connectionId}/test`, {}) as { success?: boolean; detail?: string } + : await apiClientV2.post('/connections/test-config', { type: sourceType, config: cv }) as { success?: boolean; detail?: string } + if (result?.success) { + setTestStatus('success') + setTestMessage(t('connectorInspector.test_success')) + } else { + setTestStatus('failed') + setTestMessage(result?.detail || t('connectorInspector.test_failed')) + } + } catch (e: unknown) { + setTestStatus('failed') + setTestMessage((e as { detail?: string } | null | undefined)?.detail || t('connectorInspector.test_failed')) + } + } + return ( <> -
-
+
+
{sourceType === 'file' && (
@@ -100,13 +227,31 @@ export default function ConnectorInspector({ config, onChange, readOnly = false ))}
)} )} - {sourceType !== 'file' && (
{DB_CONFIG_FIELDS[sourceType]?.map(f => (
| undefined)?.[f.key] || '')} onChange={e => { const cv2 = { ...((config.config_values as Record | undefined) || {}), [f.key]: e.target.value }; onChange('config_values', cv2); setTestStatus('idle') }} placeholder={f.placeholder} className="w-full border rounded-lg px-3 py-1.5 text-sm" />
))}
)} + {isSqlSource && (
+
+ + + {!connectionLoading && matchingConnections.length === 0 &&

{t('connectorInspector.no_saved_connections')}

} +
+
+ + + {resourceError &&

{resourceError}

} +
+
)} + {sourceType !== 'file' && !isSqlSource && !isUnsupportedSource && (
{DB_CONFIG_FIELDS[sourceType]?.map(f => (
| undefined)?.[f.key] || '')} onChange={e => { const cv2 = { ...((config.config_values as Record | undefined) || {}), [f.key]: e.target.value }; onChange('config_values', cv2); setTestStatus('idle') }} placeholder={f.placeholder} className="w-full border rounded-lg px-3 py-1.5 text-sm" />
))}
)}
-
-
+
) } diff --git a/frontend/src/pages/pipelines/connections/ConnectionsTab.test.tsx b/frontend/src/pages/pipelines/connections/ConnectionsTab.test.tsx new file mode 100644 index 00000000..b681a30a --- /dev/null +++ b/frontend/src/pages/pipelines/connections/ConnectionsTab.test.tsx @@ -0,0 +1,70 @@ +import '@/i18n' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { render, screen, fireEvent, waitFor } from '@testing-library/react' +import { apiClientV2 } from '@/api/client' +import ConnectionsTab from './ConnectionsTab' +import { useAuthStore } from '@/stores/authStore' + +vi.mock('@/api/client', () => ({ + apiClientV2: { get: vi.fn(), post: vi.fn(), delete: vi.fn() }, +})) + +describe('ConnectionsTab database connections', () => { + beforeEach(() => { + vi.clearAllMocks() + useAuthStore.setState({ user: { id: 'editor', username: 'editor', email: '', role: 'editor', is_active: true, created_at: '' } }) + }) + afterEach(() => useAuthStore.setState({ user: null, token: null })) + + it('lets viewers read connections without exposing editor-only actions', async () => { + useAuthStore.setState({ user: { ...useAuthStore.getState().user!, role: 'viewer' } }) + vi.mocked(apiClientV2.get).mockResolvedValueOnce([{ id: 'conn-1', name: 'Read-only connection', kind: 'mysql', status: 'active' }]) + render() + await screen.findByText('Read-only connection') + expect(screen.queryByRole('button', { name: /新建连接|new connection/i })).toBeNull() + expect(screen.queryByRole('button', { name: /发现数据表|discover tables/i })).toBeNull() + expect(screen.queryByRole('button', { name: /删除|delete/i })).toBeNull() + }) + + it('tests a database config before saving it', async () => { + vi.mocked(apiClientV2.get).mockResolvedValueOnce([]) + vi.mocked(apiClientV2.post) + .mockResolvedValueOnce({ success: true }) + .mockResolvedValueOnce({ id: 'conn-1' }) + .mockResolvedValueOnce([]) + render() + + await waitFor(() => expect(screen.getByRole('button', { name: /新建连接|new connection/i })).toBeTruthy()) + fireEvent.click(screen.getByRole('button', { name: /新建连接|new connection/i })) + fireEvent.change(screen.getByLabelText(/连接名称|connection name/i), { target: { value: 'Runtime MySQL' } }) + fireEvent.click(screen.getByRole('button', { name: /测试连接|test connection/i })) + + await waitFor(() => expect(screen.getByText(/连接成功|connected/i)).toBeTruthy()) + expect(apiClientV2.post).toHaveBeenCalledWith('/connections/test-config', expect.objectContaining({ type: 'mysql' })) + }) + + it('discovers tables for a saved database connection', async () => { + vi.mocked(apiClientV2.get) + .mockResolvedValueOnce([{ id: 'conn-1', name: 'Runtime MySQL', kind: 'mysql', status: 'active' }]) + .mockResolvedValueOnce({ resources: ['codex_pipeline_probe'] }) + render() + + await waitFor(() => expect(screen.getByText('Runtime MySQL')).toBeTruthy()) + fireEvent.click(screen.getByRole('button', { name: /发现数据表|discover tables/i })) + await waitFor(() => expect(screen.getByText('codex_pipeline_probe')).toBeTruthy()) + expect(apiClientV2.get).toHaveBeenCalledWith('/connections/conn-1/resources') + expect(screen.queryByRole('button', { name: /同步|sync/i })).toBeNull() + expect(screen.getByText(/Pipeline Builder/)).toBeTruthy() + }) + + it('shows a resource discovery error instead of an empty table result', async () => { + vi.mocked(apiClientV2.get) + .mockResolvedValueOnce([{ id: 'conn-1', name: 'Runtime MySQL', kind: 'mysql', status: 'active' }]) + .mockRejectedValueOnce(new Error('database unavailable')) + render() + + await waitFor(() => expect(screen.getByText('Runtime MySQL')).toBeTruthy()) + fireEvent.click(screen.getByRole('button', { name: /发现数据表|discover tables/i })) + await waitFor(() => expect(screen.getByText('database unavailable')).toBeTruthy()) + }) +}) diff --git a/frontend/src/pages/pipelines/connections/ConnectionsTab.tsx b/frontend/src/pages/pipelines/connections/ConnectionsTab.tsx index 6250a2d3..4b8b5b09 100644 --- a/frontend/src/pages/pipelines/connections/ConnectionsTab.tsx +++ b/frontend/src/pages/pipelines/connections/ConnectionsTab.tsx @@ -1,8 +1,9 @@ import { useState, useEffect, useCallback } from 'react' import { useTranslation } from 'react-i18next' import { useDropzone } from 'react-dropzone' -import { Plus, Database, FileUp, Globe, X, Loader2, RefreshCw } from 'lucide-react' +import { Plus, Database, FileUp, Globe, X, Loader2, RefreshCw, CheckCircle, XCircle } from 'lucide-react' import { apiClientV2 } from '@/api/client' +import { useAuthStore } from '@/stores/authStore' interface Connection { id: string @@ -106,12 +107,21 @@ function FileUploadZone({ files, onFilesChange }: { files: File[]; onFilesChange export default function ConnectionsTab() { const { t } = useTranslation() + const role = useAuthStore(s => s.user?.role) + const canEdit = role === 'editor' || role === 'admin' const [connections, setConnections] = useState([]) const [loading, setLoading] = useState(true) const [showForm, setShowForm] = useState(false) const [saving, setSaving] = useState(false) const [formError, setFormError] = useState('') const [syncing, setSyncing] = useState(null) + const [testing, setTesting] = useState(false) + const [testStatus, setTestStatus] = useState<'idle' | 'success' | 'failed'>('idle') + const [testMessage, setTestMessage] = useState('') + const [expandedConnection, setExpandedConnection] = useState(null) + const [resources, setResources] = useState>({}) + const [resourceErrors, setResourceErrors] = useState>({}) + const [resourceLoading, setResourceLoading] = useState(null) const [formName, setFormName] = useState('') const [formKind, setFormKind] = useState('mysql') @@ -140,9 +150,40 @@ export default function ConnectionsTab() { setFormSyncMode('snapshot') setFormFiles([]) setFormError('') + setTestStatus('idle') + setTestMessage('') + } + + const handleTest = async () => { + if (!canEdit) return + if (formKind === 'file') { + setTestStatus(formFiles.length > 0 ? 'success' : 'failed') + setTestMessage(formFiles.length > 0 ? t('connectionsTab.test_success') : t('connectionsTab.files_required_error')) + return + } + setTesting(true) + setTestStatus('idle') + setTestMessage('') + try { + const result = await apiClientV2.post('/connections/test-config', { type: formKind, config: formConfig }) as { success?: boolean; detail?: string } + if (result?.success) { + setTestStatus('success') + setTestMessage(t('connectionsTab.test_success')) + } else { + setTestStatus('failed') + setTestMessage(result?.detail || t('connectionsTab.test_failed')) + } + } catch (e: unknown) { + const err = e as { detail?: string; message?: string } | null | undefined + setTestStatus('failed') + setTestMessage(err?.detail || err?.message || t('connectionsTab.test_failed')) + } finally { + setTesting(false) + } } const handleSave = async () => { + if (!canEdit) return if (!formName.trim()) { setFormError(t('connectionsTab.name_required_error')); return } if (formKind === 'file' && formFiles.length === 0) { setFormError(t('connectionsTab.files_required_error')); return } setSaving(true) @@ -178,6 +219,7 @@ export default function ConnectionsTab() { } const handleSync = async (id: string) => { + if (!canEdit) return setSyncing(id) try { await apiClientV2.post(`/connections/${id}/sync`, {}) @@ -190,11 +232,37 @@ export default function ConnectionsTab() { } const handleDelete = async (id: string) => { + if (!canEdit) return if (!window.confirm(t('connectionsTab.confirm_delete'))) return await apiClientV2.delete(`/connections/${id}`) loadConnections() } + const discoverResources = async (id: string) => { + if (!canEdit) return + if (expandedConnection === id && resources[id]) { + setExpandedConnection(null) + return + } + setResourceLoading(id) + setResourceErrors(previous => ({ ...previous, [id]: '' })) + try { + const result = await apiClientV2.get<{ resources?: string[] }>(`/connections/${id}/resources`) + const list = Array.isArray(result) ? result as unknown as string[] : (result?.resources || []) + setResources(previous => ({ ...previous, [id]: list })) + setExpandedConnection(id) + } catch (e: unknown) { + const err = e as { detail?: string; message?: string } | null | undefined + setResourceErrors(previous => ({ + ...previous, + [id]: err?.detail || err?.message || t('connectionsTab.resources_failed'), + })) + setExpandedConnection(id) + } finally { + setResourceLoading(null) + } + } + if (loading) return
{t('common.loading')}
return ( @@ -204,15 +272,15 @@ export default function ConnectionsTab() {

{t('connectionsTab.title')}

{t('connectionsTab.subtitle')}

- + } - {showForm && ( + {canEdit && showForm && (

{t('connectionsTab.new_connection')}

@@ -222,8 +290,9 @@ export default function ConnectionsTab() {
- + setFormName(e.target.value)} placeholder={t('connectionsTab.name_ph')} @@ -287,11 +356,22 @@ export default function ConnectionsTab() {
{formError &&

{formError}

} + {testStatus !== 'idle' &&

+ {testStatus === 'success' ? : } {testMessage} +

}
+ + )} + {canEdit && (isSqlConnection ? ( + + {t('connectionsTab.pipeline_sync_hint')} + + ) : ( + + ))} + {canEdit && }
- - {statusLabel} - - - + {canEdit && expandedConnection === c.id && ( +
+ {resourceErrors[c.id] ? ( +

{resourceErrors[c.id]}

+ ) : resources[c.id]?.length ? ( +
+ {resources[c.id].map(resource => {resource})} +
+ ) :

{t('connectionsTab.no_tables')}

} +
+ )}
) })} From 4d6080c93970cfd21ea3d5b458965ca22393c5b1 Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Fri, 13 Nov 2026 08:33:55 +0800 Subject: [PATCH 20/27] fix(models): delete a model on the legacy schema without active_version_id delete_model always cleared model_configs.active_version_id, a column that only exists after the versioning migration. On a schema built by metadata.create_all that UPDATE failed. Skip it when the versioning schema isn't present; migrated databases keep their reference guards. Co-Authored-By: Claude Opus 5.5 --- backend/app/routers/models.py | 7 ++- .../tests/agent/test_model_caller_cutover.py | 57 ++++++++++++++++++- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/backend/app/routers/models.py b/backend/app/routers/models.py index 5749c667..9b68eac2 100644 --- a/backend/app/routers/models.py +++ b/backend/app/routers/models.py @@ -39,6 +39,7 @@ remediate_blocked_identity, rotate_credential, select_active_version, + versioning_schema_present, ) router = APIRouter() @@ -344,8 +345,10 @@ def delete_model(model_id: str, db: Session = Depends(get_db), _=Depends(require # RESTRICT-reference model_configs.id, so they must be cleared first — # and model_configs.active_version_id RESTRICT-references the version # row itself (the reverse direction), so that pointer must be nulled - # before the version row can go. - db.execute(text("UPDATE model_configs SET active_version_id = NULL WHERE id = :id"), {"id": model_id}) + # before the version row can go. The pointer is migration-only; the + # legacy metadata.create_all schema has no such column. + if versioning_schema_present(db): + db.execute(text("UPDATE model_configs SET active_version_id = NULL WHERE id = :id"), {"id": model_id}) db.execute(text("DELETE FROM model_credentials WHERE model_config_id = :id"), {"id": model_id}) db.execute(text("DELETE FROM model_migration_findings WHERE model_config_id = :id"), {"id": model_id}) db.execute(text("DELETE FROM model_config_versions WHERE model_config_id = :id"), {"id": model_id}) diff --git a/backend/tests/agent/test_model_caller_cutover.py b/backend/tests/agent/test_model_caller_cutover.py index 07f075aa..3aedbc2f 100644 --- a/backend/tests/agent/test_model_caller_cutover.py +++ b/backend/tests/agent/test_model_caller_cutover.py @@ -178,6 +178,57 @@ def _headers(token): return {"Authorization": f"Bearer {token}"} +@pytest.mark.parametrize("reference", [None, "agent", "prompt"]) +def test_delete_model_preserves_migrated_reference_guards(pg, reference): + from fastapi import HTTPException + from app.models.agent import Agent, AgentVersion + from app.models.agent_config import PromptGeneration + from app.models.model_version import ModelMigrationFinding + from app.routers.models import delete_model + + model_id = _seed_llm_config(pg) + _migrate_rows(pg) + version_id = pg.execute(text( + "SELECT active_version_id FROM model_configs WHERE id = :id" + ), {"id": model_id}).scalar_one() + assert version_id is not None + pg.add(ModelMigrationFinding(model_config_id=model_id, code="TEST", reason="test finding")) + if reference: + pg.add(Agent(id="delete-test-agent", owner_id="seed-user")) + pg.flush() + if reference == "agent": + schema_id = pg.execute(text( + "SELECT active_version_id FROM application_state_schema_registries WHERE application_key = 'chat-v1'" + )).scalar_one() + pg.add(AgentVersion( + agent_id="delete-test-agent", version_no=1, name="dependent", + default_model_config_version_id=version_id, default_model_name="gpt-4o", + application_state_schema_version_id=schema_id, config_hash="a" * 64, + created_by="seed-user", + )) + else: + pg.add(PromptGeneration( + agent_id="delete-test-agent", base_version_no=1, + model_config_version_id=version_id, model_name="gpt-4o", + input_hash="b" * 64, output_hash="c" * 64, requester_id="seed-user", + )) + pg.commit() + + for table in ("model_config_versions", "model_credentials", "model_migration_findings"): + assert pg.execute(text(f"SELECT count(*) FROM {table} WHERE model_config_id = :id"), {"id": model_id}).scalar_one() > 0 + if reference: + with pytest.raises(HTTPException) as denied: + delete_model(model_id, db=pg) + assert denied.value.status_code == 409 + assert denied.value.detail == "MODEL_REFERENCED" + assert pg.execute(text("SELECT active_version_id FROM model_configs WHERE id = :id"), {"id": model_id}).scalar_one() == version_id + else: + delete_model(model_id, db=pg) + assert pg.execute(text("SELECT count(*) FROM model_configs WHERE id = :id"), {"id": model_id}).scalar_one() == 0 + for table in ("model_config_versions", "model_credentials", "model_migration_findings"): + assert pg.execute(text(f"SELECT count(*) FROM {table} WHERE model_config_id = :id"), {"id": model_id}).scalar_one() == 0 + + def test_port_resolves_immutable_llm_version_and_rejects_blocked(pg): from app.services.publication.extraction_model_port import SqlExtractionModelPort @@ -277,8 +328,10 @@ def test_model_api_redacted_tagged_union_and_versioned_surface(pg): assert detail["active_version"]["version_no"] == 2 assert detail["active_version"]["behavior_hash"] != versions[0]["behavior_hash"] - # delete is RESTRICT while versions exist - assert client.delete(f"/api/v1/models/{llm_id}", headers=headers).status_code == 409 + # Owned versions alone do not make an unused model undeletable. Real + # Agent/prompt references are covered separately by the deletion matrix. + assert client.delete(f"/api/v1/models/{llm_id}", headers=headers).status_code == 204 + assert client.get(f"/api/v1/models/{llm_id}", headers=headers).status_code == 404 # viewer cannot create/update models (editor ceiling) assert client.post("/api/v1/models", json={"name": "x", "provider": "openai", "models": ["gpt-4o"]}, headers=viewer_headers).status_code == 403 From 7d7017e2cd1eac9647265daecc0d52534713219b Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Fri, 13 Nov 2026 08:33:55 +0800 Subject: [PATCH 21/27] chore(python): cap supported Python at <3.13 Declare requires-python ">=3.11,<3.13" and make the startup guard reject 3.13+ as well as <3.11, with a message that names both bounds. Co-Authored-By: Claude Opus 5.5 --- backend/pyproject.toml | 2 +- backend/scripts/check_python_version.py | 9 ++++++--- backend/tests/agent/test_python_floor.py | 16 ++++++++-------- backend/tests/test_python_version.py | 18 ++++++++++++++++++ 4 files changed, 33 insertions(+), 12 deletions(-) create mode 100644 backend/tests/test_python_version.py diff --git a/backend/pyproject.toml b/backend/pyproject.toml index c2826a3f..e41e1c44 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -2,7 +2,7 @@ name = "ontexus-backend" version = "0.1.0" description = "Ontexus backend service" -requires-python = ">=3.11" +requires-python = ">=3.11,<3.13" [tool.setuptools.packages.find] include = ["app*"] diff --git a/backend/scripts/check_python_version.py b/backend/scripts/check_python_version.py index eeec27e6..cf0fc6db 100644 --- a/backend/scripts/check_python_version.py +++ b/backend/scripts/check_python_version.py @@ -3,15 +3,18 @@ import sys MIN_SUPPORTED = (3, 11) +MAX_SUPPORTED_EXCLUSIVE = (3, 13) UNSUPPORTED_PYTHON_VERSION = "UNSUPPORTED_PYTHON_VERSION" def require_supported_python(): - if sys.version_info[:2] < MIN_SUPPORTED: + if sys.version_info[:2] < MIN_SUPPORTED or sys.version_info[:2] >= MAX_SUPPORTED_EXCLUSIVE: detected = ".".join(str(part) for part in sys.version_info[:3]) - required = ".".join(str(part) for part in MIN_SUPPORTED) + minimum = ".".join(str(part) for part in MIN_SUPPORTED) + maximum = ".".join(str(part) for part in MAX_SUPPORTED_EXCLUSIVE) raise SystemExit( - f"{UNSUPPORTED_PYTHON_VERSION}: detected Python {detected}, required >= {required}" + f"{UNSUPPORTED_PYTHON_VERSION}: detected Python {detected}, " + f"required >= {minimum}, < {maximum}" ) return True diff --git a/backend/tests/agent/test_python_floor.py b/backend/tests/agent/test_python_floor.py index 2ec9b4e9..595336d4 100644 --- a/backend/tests/agent/test_python_floor.py +++ b/backend/tests/agent/test_python_floor.py @@ -14,7 +14,7 @@ BOOTSTRAP_PATH = BACKEND_DIR / "scripts" / "bootstrap_backend.py" PYPROJECT_PATH = BACKEND_DIR / "pyproject.toml" UNSUPPORTED_PYTHON_VERSION = "UNSUPPORTED_PYTHON_VERSION" -REQUIRED_PYTHON_FLOOR = ">=3.11" +REQUIRED_PYTHON_RANGE = ">=3.11,<3.13" def _run_entrypoint(*args, env=None): @@ -59,10 +59,10 @@ def test_e0_py_red_contract(): if missing: pytest.fail("RED_E0_PY: Python floor contract missing: " + ", ".join(missing)) metadata = tomllib.loads(PYPROJECT_PATH.read_text()) - if metadata.get("project", {}).get("requires-python") != REQUIRED_PYTHON_FLOOR: + if metadata.get("project", {}).get("requires-python") != REQUIRED_PYTHON_RANGE: pytest.fail( f"RED_E0_PY: pyproject.toml must declare requires-python = " - f"{REQUIRED_PYTHON_FLOOR!r}" + f"{REQUIRED_PYTHON_RANGE!r}" ) proc = _run_guard() if proc.returncode != 0: @@ -207,8 +207,8 @@ def test_require_supported_python_accepts_current_runtime(): assert _load_guard().require_supported_python() is True -def test_require_supported_python_rejects_python_3_10_and_earlier(): - for major_minor in [(3, 10), (3, 9), (3, 8)]: +def test_require_supported_python_rejects_versions_outside_supported_range(): + for major_minor in [(3, 10), (3, 9), (3, 8), (3, 13)]: proc = subprocess.run( [ sys.executable, @@ -225,7 +225,7 @@ def test_require_supported_python_rejects_python_3_10_and_earlier(): assert UNSUPPORTED_PYTHON_VERSION in message detected = ".".join(str(part) for part in major_minor) assert f"detected Python {detected}" in message - assert "required >= 3.11" in message + assert "required >= 3.11, < 3.13" in message def test_guard_runs_without_site_packages(): @@ -248,7 +248,7 @@ def test_guard_imports_stdlib_only(): def test_pyproject_declares_python_floor(): metadata = tomllib.loads(PYPROJECT_PATH.read_text()) - assert metadata["project"]["requires-python"] == REQUIRED_PYTHON_FLOOR + assert metadata["project"]["requires-python"] == REQUIRED_PYTHON_RANGE def test_packaging_build_evaluates_and_emits_python_floor(tmp_path): @@ -278,4 +278,4 @@ def test_packaging_build_evaluates_and_emits_python_floor(tmp_path): assert len(installed) == 1 metadata = installed[0]["metadata"] assert metadata["name"] == "ontexus-backend" - assert metadata["requires_python"] == REQUIRED_PYTHON_FLOOR + assert set(metadata["requires_python"].split(",")) == set(REQUIRED_PYTHON_RANGE.split(",")) diff --git a/backend/tests/test_python_version.py b/backend/tests/test_python_version.py new file mode 100644 index 00000000..a0fdd391 --- /dev/null +++ b/backend/tests/test_python_version.py @@ -0,0 +1,18 @@ +import pytest + +from scripts import check_python_version + + +@pytest.mark.parametrize("version", [(3, 11, 0), (3, 12, 0)]) +def test_supported_python_versions_are_accepted(monkeypatch, version): + monkeypatch.setattr(check_python_version.sys, "version_info", version) + + assert check_python_version.require_supported_python() is True + + +@pytest.mark.parametrize("version", [(3, 10, 0), (3, 13, 0)]) +def test_python_versions_outside_supported_range_are_rejected(monkeypatch, version): + monkeypatch.setattr(check_python_version.sys, "version_info", version) + + with pytest.raises(SystemExit, match="UNSUPPORTED_PYTHON_VERSION"): + check_python_version.require_supported_python() From 8bf5e514851354c9d430bd357bad3acd3fdb9d46 Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Fri, 13 Nov 2026 08:33:55 +0800 Subject: [PATCH 22/27] chore: point in-network Chroma clients at port 8000; README React 19 Inside the compose network Chroma listens on its container port 8000; 8001 is only the host-side mapping. Set CHROMA_PORT=8000 for the backend and worker services and in .env.example. README now lists React 19. Co-Authored-By: Claude Opus 5.5 --- .env.example | 4 ++-- README.md | 2 +- docker-compose.v2.yml | 11 +++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 4035784b..95db94b5 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,6 @@ MINIO_SECRET_KEY=minioadmin MINIO_USE_SSL=false # v2 — ChromaDB +# Docker network uses port 8000; host-side clients use localhost:8001. CHROMA_HOST=chromadb -CHROMA_PORT=8001 - +CHROMA_PORT=8000 diff --git a/README.md b/README.md index f0a289ce..01d5857a 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ In Ontexus, every ontology is made of these building blocks: | Layer | Technology | |---|---| -| Frontend | React 18, TypeScript, Vite, Tailwind CSS, Cytoscape.js | +| Frontend | React 19, TypeScript, Vite, Tailwind CSS, Cytoscape.js | | Backend | FastAPI, SQLAlchemy, Alembic | | Metadata DB | SQLite (dev) / PostgreSQL (prod) | | Object storage | MinIO (optional, local-file fallback) | diff --git a/docker-compose.v2.yml b/docker-compose.v2.yml index a5f66c9d..85c422da 100644 --- a/docker-compose.v2.yml +++ b/docker-compose.v2.yml @@ -78,6 +78,9 @@ services: build: ./backend ports: ["8000:8000"] env_file: .env + # In-network clients use the container port, not the host's 8001 mapping. + environment: &chroma-environment + CHROMA_PORT: "8000" depends_on: migration: condition: service_completed_successfully @@ -103,6 +106,7 @@ services: env_file: .env profiles: ["refresh"] environment: + <<: *chroma-environment REFRESH_WORKER_CONCURRENCY: ${REFRESH_WORKER_CONCURRENCY:-2} REFRESH_WORKER_PREFETCH_MULTIPLIER: ${REFRESH_WORKER_PREFETCH_MULTIPLIER:-1} REFRESH_SOFT_TIME_LIMIT_SECONDS: ${REFRESH_SOFT_TIME_LIMIT_SECONDS:-300} @@ -128,6 +132,7 @@ services: env_file: .env profiles: ["refresh"] environment: + <<: *chroma-environment REFRESH_WORKER_CONCURRENCY: ${REFRESH_WORKER_CONCURRENCY:-2} REFRESH_WORKER_PREFETCH_MULTIPLIER: ${REFRESH_WORKER_PREFETCH_MULTIPLIER:-1} REFRESH_SOFT_TIME_LIMIT_SECONDS: ${REFRESH_SOFT_TIME_LIMIT_SECONDS:-300} @@ -153,6 +158,7 @@ services: env_file: .env profiles: ["refresh"] environment: + <<: *chroma-environment REFRESH_WORKER_CONCURRENCY: ${REFRESH_WORKER_CONCURRENCY:-2} REFRESH_WORKER_PREFETCH_MULTIPLIER: ${REFRESH_WORKER_PREFETCH_MULTIPLIER:-1} REFRESH_SOFT_TIME_LIMIT_SECONDS: ${REFRESH_SOFT_TIME_LIMIT_SECONDS:-300} @@ -178,6 +184,7 @@ services: env_file: .env profiles: ["refresh"] environment: + <<: *chroma-environment REFRESH_WORKER_CONCURRENCY: ${REFRESH_WORKER_CONCURRENCY:-2} REFRESH_WORKER_PREFETCH_MULTIPLIER: ${REFRESH_WORKER_PREFETCH_MULTIPLIER:-1} REFRESH_SOFT_TIME_LIMIT_SECONDS: ${REFRESH_SOFT_TIME_LIMIT_SECONDS:-300} @@ -202,6 +209,7 @@ services: build: ./backend env_file: .env profiles: ["artifact"] + environment: *chroma-environment depends_on: migration: condition: service_completed_successfully @@ -220,6 +228,7 @@ services: build: ./backend env_file: .env profiles: ["agent"] + environment: *chroma-environment depends_on: migration: condition: service_completed_successfully @@ -238,6 +247,7 @@ services: build: ./backend env_file: .env profiles: ["housekeeping"] + environment: *chroma-environment depends_on: migration: condition: service_completed_successfully @@ -255,6 +265,7 @@ services: celery_beat: build: ./backend env_file: .env + environment: *chroma-environment depends_on: migration: condition: service_completed_successfully From 3b39ad33d669c7bfc08cc043f40fab1ddf5f6be0 Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Fri, 13 Nov 2026 08:33:55 +0800 Subject: [PATCH 23/27] test: take the snapshot-freshness clock per test, not at import FIXED_NOW was captured when the module was imported, so in a long run the fixtures could drift into a different lag bucket by the time a test ran. Each fixture and test now reads the clock itself. Co-Authored-By: Claude Opus 5.5 --- .../tests/runtime/test_snapshot_freshness.py | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/backend/tests/runtime/test_snapshot_freshness.py b/backend/tests/runtime/test_snapshot_freshness.py index a8240004..3f83e6db 100644 --- a/backend/tests/runtime/test_snapshot_freshness.py +++ b/backend/tests/runtime/test_snapshot_freshness.py @@ -40,14 +40,9 @@ from app.services.runtime.snapshots import get_snapshot, materialize_refresh_snapshot from app.services.v2.incremental.operations import get_last_successful_refresh_run -# Captured once at import so every case in this module reasons about the -# same instant: `test_fresh_snapshot_exposes_cursor_lag_and_lineage` needs -# an exact lag match against a `compute_snapshot_freshness(..., now=FIXED_NOW)` -# call, while `evaluate_fixture_runtime_freshness` exercises the real -# `RuntimeService` (which reads real wall-clock time internally) moments -# later in the same test process — negligible drift against this module's -# hour/day-scale lag buckets. -FIXED_NOW = datetime.now(timezone.utc) +def _now() -> datetime: + """Capture wall-clock time per fixture/test, not once at module import.""" + return datetime.now(timezone.utc) DOMAIN = "00000000-0000-0000-0000-0000000000f0" ONTOLOGY_ID = "ontology-freshness-001" @@ -167,17 +162,18 @@ def _seed_completed_run(db, *, dataset_version_id: str, pipeline_run_id: str, re db.commit() -def materialize_refresh_fixture(db, *, refresh_run_id: str): +def materialize_refresh_fixture(db, *, refresh_run_id: str, now: datetime | None = None): """Seed one governed dataset/pipeline pair plus the named `RefreshRun` and materialize the snapshot it backs. `refresh_run_id` selects the case shape: a plain success id gets a - fresh cursor/lag anchored to `FIXED_NOW`; "soft-stale"/"hard-stale" get + fresh cursor/lag anchored to the supplied/current test time; "soft-stale"/"hard-stale" get a cursor whose lag already exceeds the shared default policy's soft/hard thresholds; "unknown-cursor" gets a successful run with no recorded cursor at all. """ release = _ensure_release(db) + now = now or _now() dataset_version_id = f"dv-{refresh_run_id}" # "refresh-success-001" is this module's canonical fresh case, matching # the exact `pipeline_run_ids == ["pipeline-run-001"]` assertion below. @@ -198,7 +194,7 @@ def materialize_refresh_fixture(db, *, refresh_run_id: str): ) else: lag = _CASE_LAG_SECONDS[refresh_run_id] - observed_at = FIXED_NOW - timedelta(seconds=lag) + observed_at = now - timedelta(seconds=lag) run = RefreshRun( id=refresh_run_id, source_id="source-freshness-001", resource="orders", policy="batch", config_version=1, cursor_contract="watermark_primary_key", @@ -296,9 +292,10 @@ def get_action_plan(db, plan_id): def test_fresh_snapshot_exposes_cursor_lag_and_lineage(db): - snapshot = materialize_refresh_fixture(db, refresh_run_id="refresh-success-001") + now = _now() + snapshot = materialize_refresh_fixture(db, refresh_run_id="refresh-success-001", now=now) view = compute_snapshot_freshness( - snapshot, now=FIXED_NOW, policy=FreshnessPolicy(max_lag_seconds=300, stale_action="deny"), + snapshot, now=now, policy=FreshnessPolicy(max_lag_seconds=300, stale_action="deny"), ) assert view.state == "fresh" assert view.lag_seconds == 120 @@ -328,7 +325,7 @@ def test_unrelated_refresh_run_cannot_stamp_freshness_onto_unconnected_dataset_v idempotency_key="idem-refresh-unrelated-001", cursor_after_json={ "watermark": None, "primary_key": "999", "opaque_value": None, - "observed_at": FIXED_NOW.isoformat(), + "observed_at": _now().isoformat(), }, fencing_token=1, lag_seconds=5, pipeline_run_id="pr-unrelated-001", ) @@ -429,6 +426,7 @@ def test_investigate_succeeds_with_real_production_shaped_refresh_provenance(db, refresh_run_id = "refresh-provenance-shape-001" dataset_version_id = f"dv-{refresh_run_id}" pipeline_run_id = f"pr-{refresh_run_id}" + now = _now() # The exact dict shape production code builds — no source_type/locator/ # content_hash key anywhere (see polling.py:630-638, event_ingest.py:668-678). @@ -439,7 +437,7 @@ def test_investigate_succeeds_with_real_production_shaped_refresh_provenance(db, "config_version": 1, "cursor_contract": "watermark_primary_key", "cursor_outcome": "advanced", - "source_observed_at": (FIXED_NOW - timedelta(seconds=60)).isoformat(), + "source_observed_at": (now - timedelta(seconds=60)).isoformat(), } pipeline = Pipeline(id=f"pipeline-{pipeline_run_id}", name=pipeline_run_id, spec={}) @@ -468,7 +466,7 @@ def test_investigate_succeeds_with_real_production_shaped_refresh_provenance(db, idempotency_key=f"idem-{refresh_run_id}", cursor_after_json={ "watermark": None, "primary_key": "100", "opaque_value": None, - "observed_at": (FIXED_NOW - timedelta(seconds=60)).isoformat(), + "observed_at": (now - timedelta(seconds=60)).isoformat(), }, fencing_token=1, lag_seconds=60, pipeline_run_id=pipeline_run_id, source_provenance=dict(provenance), From cc201c46bce9716d230a8a2203a86a131b92c288 Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Fri, 13 Nov 2026 08:34:29 +0800 Subject: [PATCH 24/27] refactor(frontend): drop synchronous setState in effects react-hooks/set-state-in-effect flagged resets inside effects in ConversationPanel, V2DefinitionEditor and the four runtime pages. Per-id state is now keyed by the id it belongs to, or the content component is remounted with a key, so the effects only start loads. Effects also ignore results that arrive after a newer request, and the tests cover switching ids. Co-Authored-By: Claude Opus 5.5 --- .../application/ConversationPanel.test.tsx | 14 ++++ .../agents/application/ConversationPanel.tsx | 12 +-- .../detail/tabs/V2DefinitionEditor.test.tsx | 25 ++++++ .../detail/tabs/V2DefinitionEditor.tsx | 12 ++- .../src/pages/runtime/ActionPlanPage.test.tsx | 26 +++++- frontend/src/pages/runtime/ActionPlanPage.tsx | 13 ++- .../pages/runtime/ReconciliationPage.test.tsx | 25 +++++- .../src/pages/runtime/ReconciliationPage.tsx | 11 ++- .../runtime/RefreshOperationsPage.test.tsx | 26 +++++- .../pages/runtime/RefreshOperationsPage.tsx | 13 ++- .../runtime/RuntimeInvestigationPage.test.tsx | 22 ++++- .../runtime/RuntimeInvestigationPage.tsx | 80 ++++++++++++++----- 12 files changed, 231 insertions(+), 48 deletions(-) create mode 100644 frontend/src/pages/ontologies/detail/tabs/V2DefinitionEditor.test.tsx diff --git a/frontend/src/pages/agents/application/ConversationPanel.test.tsx b/frontend/src/pages/agents/application/ConversationPanel.test.tsx index 692c6eb8..8153f06b 100644 --- a/frontend/src/pages/agents/application/ConversationPanel.test.tsx +++ b/frontend/src/pages/agents/application/ConversationPanel.test.tsx @@ -14,6 +14,20 @@ function renderPanel(messages: AgentMessage[], stream: StreamState = initialStre } describe('P4A-CONVERSATION', () => { + it('hides prior-turn evidence synchronously when the turn changes', async () => { + const get = vi.spyOn(apiClient, 'get').mockImplementation(url => url.includes('/old/') + ? Promise.resolve({ items: [{ sequence: 1, event_type: 'final_response', payload: { audit_event_id: 'old-audit' } }] }) as never + : new Promise(() => {})) + const props = { messages: [], stream: { ...initialStreamState, terminal: true }, clarification: null, + pendingApprovalId: null, onSend: () => {}, onAnswerClarification: () => {}, onApprovalResolved: () => {}, onRetry: () => {} } + try { + const view = render() + await screen.findByText('old-audit') + view.rerender() + expect(screen.queryByTestId('journey-evidence')).toBeNull() + } finally { get.mockRestore() } + }) + it('strips the legacy canned "Answer for …" prefix from persisted messages', () => { const messages: AgentMessage[] = [{ id: 'm-1', session_id: 's-1', turn_id: 't-1', role: 'assistant', diff --git a/frontend/src/pages/agents/application/ConversationPanel.tsx b/frontend/src/pages/agents/application/ConversationPanel.tsx index ab7d5b23..b81ee299 100644 --- a/frontend/src/pages/agents/application/ConversationPanel.tsx +++ b/frontend/src/pages/agents/application/ConversationPanel.tsx @@ -88,7 +88,8 @@ export default function ConversationPanel({ const { t } = useTranslation() const [draft, setDraft] = useState('') const [clarificationAnswer, setClarificationAnswer] = useState('') - const [journeyEvents, setJourneyEvents] = useState([]) + const [journey, setJourney] = useState<{ turnId: string; events: RuntimeEventRecord[] } | null>(null) + const journeyEvents = journey?.turnId === turnId ? journey.events : [] const [showGovernedPlan, setShowGovernedPlan] = useState(false) // Persisted turn events back the always-visible business-journey @@ -98,11 +99,10 @@ export default function ConversationPanel({ // a terminal SSE/polling state. useEffect(() => { let cancelled = false - void Promise.resolve().then(() => { if (!cancelled) setJourneyEvents([]) }) if (!turnId) return () => { cancelled = true } const fetchEvents = () => { apiClient.get<{ items: RuntimeEventRecord[] }>(`/agent-turns/${turnId}/events?limit=100`) - .then(res => { if (!cancelled) setJourneyEvents(Array.isArray(res.items) ? res.items : []) }) + .then(res => { if (!cancelled) setJourney({ turnId, events: Array.isArray(res.items) ? res.items : [] }) }) .catch(() => {}) } fetchEvents() @@ -151,20 +151,20 @@ export default function ConversationPanel({ // `app.services.runtime.answer_stream`) — polled on its own fast cadence // rather than piggybacking on the slower journeyEvents poll above, since // this is what actually needs to feel live. - const [liveAnswerText, setLiveAnswerText] = useState('') + const [liveAnswer, setLiveAnswer] = useState<{ turnId: string; text: string } | null>(null) useEffect(() => { - setLiveAnswerText('') if (!turnId || !isThinking) return let cancelled = false const poll = () => { apiClient.get<{ text: string } | null>(`/agent-turns/${turnId}/answer-stream`) - .then(res => { if (!cancelled && res) setLiveAnswerText(res.text) }) + .then(res => { if (!cancelled && res) setLiveAnswer({ turnId, text: res.text }) }) .catch(() => {}) } poll() const interval = setInterval(poll, ANSWER_STREAM_POLL_INTERVAL_MS) return () => { cancelled = true; clearInterval(interval) } }, [turnId, isThinking]) + const liveAnswerText = isThinking && liveAnswer?.turnId === turnId ? liveAnswer.text : '' const submit = () => { if (!draft.trim()) return diff --git a/frontend/src/pages/ontologies/detail/tabs/V2DefinitionEditor.test.tsx b/frontend/src/pages/ontologies/detail/tabs/V2DefinitionEditor.test.tsx new file mode 100644 index 00000000..6c21e983 --- /dev/null +++ b/frontend/src/pages/ontologies/detail/tabs/V2DefinitionEditor.test.tsx @@ -0,0 +1,25 @@ +import '@/i18n' +import { act, fireEvent, render, screen } from '@testing-library/react' +import { expect, it, vi } from 'vitest' +import { apiClientV2 } from '@/api/client' +import V2DefinitionEditor from './V2DefinitionEditor' + +it('clears selection and ignores an old save reload after switching ontology', async () => { + let resolveSave!: () => void + const save = new Promise(resolve => { resolveSave = resolve }) + const get = vi.spyOn(apiClientV2, 'get').mockImplementation(url => Promise.resolve([ + { id: 'definition', name: url.includes('/old/') ? 'Old rule' : 'New rule', expression: {} }, + ]) as never) + const patch = vi.spyOn(apiClientV2, 'patch').mockReturnValue(save) + try { + const view = render() + fireEvent.click(await screen.findByText('Old rule')) + fireEvent.click(screen.getByTestId('v2-logic-save')) + view.rerender() + await screen.findByText('New rule') + await act(async () => { resolveSave(); await save }) + expect(screen.queryByText('Old rule')).toBeNull() + expect(screen.getByText('New rule')).toBeTruthy() + expect(screen.queryByTestId('v2-logic-save')).toBeNull() + } finally { get.mockRestore(); patch.mockRestore() } +}) diff --git a/frontend/src/pages/ontologies/detail/tabs/V2DefinitionEditor.tsx b/frontend/src/pages/ontologies/detail/tabs/V2DefinitionEditor.tsx index 5a01dab2..15effc2e 100644 --- a/frontend/src/pages/ontologies/detail/tabs/V2DefinitionEditor.tsx +++ b/frontend/src/pages/ontologies/detail/tabs/V2DefinitionEditor.tsx @@ -22,6 +22,10 @@ interface Props { } export default function V2DefinitionEditor({ ontologyId, kind }: Props) { + return +} + +function DefinitionEditorContent({ ontologyId, kind }: Props) { const { t } = useTranslation() const [items, setItems] = useState([]) const [selected, setSelected] = useState(null) @@ -39,7 +43,13 @@ export default function V2DefinitionEditor({ ontologyId, kind }: Props) { } } - useEffect(() => { void load() }, [ontologyId, kind]) // eslint-disable-line react-hooks/exhaustive-deps + useEffect(() => { + let cancelled = false + void apiClientV2.get(`/ontologies/${ontologyId}/${kind}`) + .then(rows => { if (!cancelled) setItems(Array.isArray(rows) ? rows : []) }) + .catch(() => { if (!cancelled) setError(t('v2def.load_failed')) }) + return () => { cancelled = true } + }, [ontologyId, kind, t]) const select = (item: Definition) => { setSelected(item) diff --git a/frontend/src/pages/runtime/ActionPlanPage.test.tsx b/frontend/src/pages/runtime/ActionPlanPage.test.tsx index 1094576e..4d5647d5 100644 --- a/frontend/src/pages/runtime/ActionPlanPage.test.tsx +++ b/frontend/src/pages/runtime/ActionPlanPage.test.tsx @@ -1,7 +1,8 @@ import '@/i18n' -import { MemoryRouter, Route, Routes } from 'react-router-dom' -import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest' -import { render, screen, waitFor } from '@testing-library/react' +import { Link, MemoryRouter, Route, Routes } from 'react-router-dom' +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest' +import { act, render, screen, waitFor } from '@testing-library/react' +import { runtimeApi } from '@/api/runtime' import userEvent from '@testing-library/user-event' import { http, HttpResponse } from 'msw' import { setupServer } from 'msw/node' @@ -50,6 +51,25 @@ function renderPage() { } describe('ActionPlanPage', () => { + it('does not replace the current plan with a late prior-route response', async () => { + type Plan = Awaited> + let resolveOld!: (value: Plan) => void + const oldResponse = new Promise(resolve => { resolveOld = resolve }) + const get = vi.spyOn(runtimeApi, 'getActionPlan').mockImplementation(id => + id === 'old' ? oldResponse : Promise.resolve({ ...PLAN, id, plan_hash: 'new-hash' } as Plan)) + const status = vi.spyOn(runtimeApi, 'getExecutionStatus').mockResolvedValue({ status: 'not_started', plan_id: 'new', correlation_id: 'test' }) + try { + render( + next plan + } /> + ) + await userEvent.click(screen.getByText('next plan')) + expect(await screen.findByTestId('plan-hash')).toHaveTextContent('new-hash') + await act(async () => { resolveOld(PLAN as Plan); await oldResponse }) + expect(screen.getByTestId('plan-hash')).toHaveTextContent('new-hash') + } finally { get.mockRestore(); status.mockRestore() } + }) + it('shows the exact plan hash, approves, then executes and shows the receipt', async () => { let approveBody: unknown = null let executeBody: unknown = null diff --git a/frontend/src/pages/runtime/ActionPlanPage.tsx b/frontend/src/pages/runtime/ActionPlanPage.tsx index eba6c11a..bbeee53b 100644 --- a/frontend/src/pages/runtime/ActionPlanPage.tsx +++ b/frontend/src/pages/runtime/ActionPlanPage.tsx @@ -17,8 +17,12 @@ function isStarted(receipt: ExecutionReceipt | ExecutionNotStarted | null): rece * hash itself; every write is the backend's own governed decision. */ export default function ActionPlanPage() { - const { t } = useTranslation() const { planId } = useParams<{ planId: string }>() + return +} + +function ActionPlanContent({ planId }: { planId?: string }) { + const { t } = useTranslation() const [plan, setPlan] = useState(null) const [receipt, setReceipt] = useState(null) const [execution, setExecution] = useState(null) @@ -27,12 +31,13 @@ export default function ActionPlanPage() { const load = useCallback(() => { if (!planId) return - setError('') - runtimeApi.getActionPlan(planId).then(setPlan).catch(() => setError(t('runtime.plan.load_failed', 'Failed to load the action plan'))) + runtimeApi.getActionPlan(planId) + .then(value => { setPlan(value); setError('') }) + .catch(() => setError(t('runtime.plan.load_failed', 'Failed to load the action plan'))) runtimeApi.getExecutionStatus(planId).then(setExecution).catch(() => {}) }, [planId, t]) - useEffect(() => { load() }, [load]) + useEffect(() => { void load() }, [load]) const approve = async () => { if (!planId || !plan) return diff --git a/frontend/src/pages/runtime/ReconciliationPage.test.tsx b/frontend/src/pages/runtime/ReconciliationPage.test.tsx index 42d3f8d3..bbcd9592 100644 --- a/frontend/src/pages/runtime/ReconciliationPage.test.tsx +++ b/frontend/src/pages/runtime/ReconciliationPage.test.tsx @@ -1,7 +1,8 @@ import '@/i18n' -import { MemoryRouter, Route, Routes } from 'react-router-dom' -import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest' -import { render, screen } from '@testing-library/react' +import { Link, MemoryRouter, Route, Routes } from 'react-router-dom' +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest' +import { act, render, screen } from '@testing-library/react' +import { runtimeApi } from '@/api/runtime' import userEvent from '@testing-library/user-event' import { http, HttpResponse } from 'msw' import { setupServer } from 'msw/node' @@ -65,6 +66,24 @@ const ROLLBACK_PLAN = { } describe('ReconciliationPage', () => { + it('does not show a late case response after navigating to another case', async () => { + let resolveOld!: (value: typeof UNKNOWN_CASE) => void + const oldResponse = new Promise(resolve => { resolveOld = resolve }) + const get = vi.spyOn(runtimeApi, 'getReconciliation').mockImplementation(id => + id === 'recon-001' ? oldResponse : Promise.resolve({ ...UNKNOWN_CASE, id, execution_id: 'exec-002' })) + try { + render( + next case + } /> + ) + await userEvent.click(screen.getByText('next case')) + expect(await screen.findByText('exec-002')).toBeTruthy() + await act(async () => { resolveOld(UNKNOWN_CASE); await oldResponse }) + expect(screen.getByText('exec-002')).toBeTruthy() + expect(screen.queryByText('exec-001')).toBeNull() + } finally { get.mockRestore() } + }) + it('renders an UNKNOWN case with its unknown reason and next action', async () => { server.use( http.get('*/api/v2/runtime/reconciliations/recon-001', () => HttpResponse.json(UNKNOWN_CASE)), diff --git a/frontend/src/pages/runtime/ReconciliationPage.tsx b/frontend/src/pages/runtime/ReconciliationPage.tsx index 38fd9dba..0eb9a2a8 100644 --- a/frontend/src/pages/runtime/ReconciliationPage.tsx +++ b/frontend/src/pages/runtime/ReconciliationPage.tsx @@ -16,8 +16,12 @@ import type { ActionPlan, ReconciliationCase } from '@/types/runtime' * that invariant, not a field the server returns. */ export default function ReconciliationPage() { - const { t } = useTranslation() const { reconciliationId } = useParams<{ reconciliationId: string }>() + return +} + +function ReconciliationContent({ reconciliationId }: { reconciliationId?: string }) { + const { t } = useTranslation() const [reconciliation, setReconciliation] = useState(null) const [rollbackPlan, setRollbackPlan] = useState(null) const [error, setError] = useState('') @@ -25,13 +29,12 @@ export default function ReconciliationPage() { const load = useCallback(() => { if (!reconciliationId) return - setError('') runtimeApi.getReconciliation(reconciliationId) - .then(setReconciliation) + .then(value => { setReconciliation(value); setError('') }) .catch(() => setError(t('runtime.reconciliation.load_failed', 'Failed to load the reconciliation case'))) }, [reconciliationId, t]) - useEffect(() => { load() }, [load]) + useEffect(() => { void load() }, [load]) const createRollback = async () => { if (!reconciliation) return diff --git a/frontend/src/pages/runtime/RefreshOperationsPage.test.tsx b/frontend/src/pages/runtime/RefreshOperationsPage.test.tsx index 1bf0e667..65a9db02 100644 --- a/frontend/src/pages/runtime/RefreshOperationsPage.test.tsx +++ b/frontend/src/pages/runtime/RefreshOperationsPage.test.tsx @@ -1,7 +1,8 @@ import '@/i18n' -import { MemoryRouter, Route, Routes } from 'react-router-dom' -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' -import { render, screen, waitFor } from '@testing-library/react' +import { Link, MemoryRouter, Route, Routes } from 'react-router-dom' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { act, render, screen, waitFor } from '@testing-library/react' +import { refreshApi } from '@/api/refresh' import userEvent from '@testing-library/user-event' import { http, HttpResponse } from 'msw' import { setupServer } from 'msw/node' @@ -56,6 +57,25 @@ function baseStatus(run: RefreshRunView, overrides: Partial = {}) } describe('RefreshOperationsPage', () => { + it('does not expose a prior source run after a delayed response', async () => { + let resolveOld!: (value: RefreshStatus) => void + const oldResponse = new Promise(resolve => { resolveOld = resolve }) + const get = vi.spyOn(refreshApi, 'getStatus').mockImplementation(id => id === 'old' ? oldResponse : + Promise.resolve(baseStatus(baseRun({ run_id: 'current-run' }), { source_id: id }))) + const schedule = vi.spyOn(refreshApi, 'getSchedule').mockRejectedValue(new Error('not configured')) + try { + render( + next source + } /> + ) + await userEvent.click(screen.getByText('next source')) + expect(await screen.findByText('current-run')).toBeTruthy() + await act(async () => { resolveOld(baseStatus(baseRun({ run_id: 'old-run' }))); await oldResponse }) + expect(screen.getByText('current-run')).toBeTruthy() + expect(screen.queryByText('old-run')).toBeNull() + } finally { get.mockRestore(); schedule.mockRestore() } + }) + it('loads the persisted schedule timezone and business calendar on mount', async () => { server.use( http.get('*/api/v2/refresh/sources/source-001/status', () => HttpResponse.json(baseStatus(baseRun({})))), diff --git a/frontend/src/pages/runtime/RefreshOperationsPage.tsx b/frontend/src/pages/runtime/RefreshOperationsPage.tsx index 697f1c59..85d00667 100644 --- a/frontend/src/pages/runtime/RefreshOperationsPage.tsx +++ b/frontend/src/pages/runtime/RefreshOperationsPage.tsx @@ -30,8 +30,12 @@ function cursorLabel(cursor: Record | null | undefined): string * never a distinct error state and never rollback language. */ export default function RefreshOperationsPage() { - const { t } = useTranslation() const { sourceId } = useParams<{ sourceId: string }>() + return +} + +function RefreshOperationsContent({ sourceId }: { sourceId?: string }) { + const { t } = useTranslation() const [status, setStatus] = useState(null) const [error, setError] = useState('') const [busy, setBusy] = useState(false) @@ -44,11 +48,12 @@ export default function RefreshOperationsPage() { const load = useCallback(() => { if (!sourceId) return - setError('') - refreshApi.getStatus(sourceId).then(setStatus).catch(() => setError(t('runtime.refresh.load_failed', 'Failed to load refresh status'))) + refreshApi.getStatus(sourceId) + .then(value => { setStatus(value); setError('') }) + .catch(() => setError(t('runtime.refresh.load_failed', 'Failed to load refresh status'))) }, [sourceId, t]) - useEffect(() => { load() }, [load]) + useEffect(() => { void load() }, [load]) useEffect(() => { if (!sourceId) return diff --git a/frontend/src/pages/runtime/RuntimeInvestigationPage.test.tsx b/frontend/src/pages/runtime/RuntimeInvestigationPage.test.tsx index 668cb648..dd9d661d 100644 --- a/frontend/src/pages/runtime/RuntimeInvestigationPage.test.tsx +++ b/frontend/src/pages/runtime/RuntimeInvestigationPage.test.tsx @@ -1,7 +1,8 @@ import '@/i18n' import { MemoryRouter } from 'react-router-dom' -import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest' -import { render, screen } from '@testing-library/react' +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest' +import { act, fireEvent, render, screen } from '@testing-library/react' +import { runtimeApi } from '@/api/runtime' import userEvent from '@testing-library/user-event' import { http, HttpResponse } from 'msw' import { setupServer } from 'msw/node' @@ -26,6 +27,23 @@ function renderPage() { } describe('RuntimeInvestigationPage', () => { + it('ignores an auto-run result after the operator changes its inputs', async () => { + type Result = Awaited> + let finish!: (value: Result) => void + const pending = new Promise(resolve => { finish = resolve }) + const investigate = vi.spyOn(runtimeApi, 'investigate').mockReturnValue(pending) + try { + renderPage() + fireEvent.change(screen.getByTestId('investigate-snapshot-id'), { target: { value: 'new-snapshot' } }) + await act(async () => { + finish({ decision: 'ALLOW', reason_code: 'ALLOW', result: [], evidence_citations: [], rule_outcome: [] } as unknown as Result) + await pending + }) + expect(screen.queryByTestId('investigation-result')).toBeNull() + expect(screen.getByTestId('run-investigation')).not.toBeDisabled() + } finally { investigate.mockRestore() } + }) + it('shows snapshot evidence and an ALLOW decision', async () => { server.use( http.post('*/api/v2/runtime/investigate', () => diff --git a/frontend/src/pages/runtime/RuntimeInvestigationPage.tsx b/frontend/src/pages/runtime/RuntimeInvestigationPage.tsx index b9128cd6..ab2cd258 100644 --- a/frontend/src/pages/runtime/RuntimeInvestigationPage.tsx +++ b/frontend/src/pages/runtime/RuntimeInvestigationPage.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { useSearchParams } from 'react-router-dom' import { runtimeApi } from '@/api/runtime' @@ -15,35 +15,56 @@ import type { InvestigationResult } from '@/types/runtime' export default function RuntimeInvestigationPage() { const { t } = useTranslation() const [searchParams] = useSearchParams() - const [snapshotId, setSnapshotId] = useState(searchParams.get('snapshot_id') ?? '') - const [ontologyId, setOntologyId] = useState(searchParams.get('ontology_id') ?? '') + const initialSnapshotId = searchParams.get('snapshot_id') ?? '' + const initialOntologyId = searchParams.get('ontology_id') ?? '' + const [snapshotId, setSnapshotId] = useState(initialSnapshotId) + const [ontologyId, setOntologyId] = useState(initialOntologyId) const [query, setQuery] = useState('') const [result, setResult] = useState(null) const [error, setError] = useState('') - const [loading, setLoading] = useState(false) + const [loading, setLoading] = useState(Boolean(initialSnapshotId.trim() && initialOntologyId.trim())) + const generation = useRef(0) + + const invalidateResult = () => { + generation.current += 1 + setResult(null) + setError('') + setLoading(false) + } + + const requestInvestigation = () => runtimeApi.investigate({ + semantic_snapshot_id: snapshotId, + ontology_id: ontologyId, + query: query || undefined, + }) + + const structuredDenial = (err: unknown): InvestigationResult | null => { + const body = err as Partial & { reason_code?: string } + return body && typeof body === 'object' && 'decision' in body + ? body as InvestigationResult + : null + } const runInvestigation = async () => { + const request = ++generation.current setLoading(true) setError('') try { - const res = await runtimeApi.investigate({ - semantic_snapshot_id: snapshotId, - ontology_id: ontologyId, - query: query || undefined, - }) - setResult(res) + const value = await requestInvestigation() + if (request === generation.current) setResult(value) } catch (err) { + if (request !== generation.current) return // A structured denial from the credential layer (401/403) arrives as // a thrown error body, not a 2xx `InvestigationResult` — surface it // the same way a 200 DENY result would render. - const body = err as Partial & { reason_code?: string } - if (body && typeof body === 'object' && 'decision' in body) { - setResult(body as InvestigationResult) + const denial = structuredDenial(err) + if (denial) { + setResult(denial) } else { setError(t('runtime.investigate.load_failed', 'Investigation request failed')) } } finally { - setLoading(false) + if (request === generation.current) setLoading(false) } } @@ -52,8 +73,31 @@ export default function RuntimeInvestigationPage() { // query params). Otherwise every page open would fire an investigation // with empty required fields, which the server always DENYs — producing // a real denial record before the operator has entered anything. + useEffect(() => { + if (!snapshotId.trim() || !ontologyId.trim()) return + let cancelled = false + const request = ++generation.current + const autoRun = async () => { + try { + const res = await requestInvestigation() + if (!cancelled && request === generation.current) setResult(res) + } catch (err) { + const denial = structuredDenial(err) + if (cancelled || request !== generation.current) return + if (denial) { + setResult(denial) + } else { + setError(t('runtime.investigate.load_failed', 'Investigation request failed')) + } + } finally { + if (!cancelled && request === generation.current) setLoading(false) + } + } + void autoRun() + return () => { cancelled = true } + // The auto-run intentionally uses only the initial deep-link values. // eslint-disable-next-line react-hooks/exhaustive-deps - useEffect(() => { if (snapshotId.trim() && ontologyId.trim()) void runInvestigation() }, []) + }, []) return (
@@ -62,17 +106,17 @@ export default function RuntimeInvestigationPage() {
From ceb148fa32e6a37dbe1614a408b1f7112b9fda37 Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Fri, 13 Nov 2026 08:34:52 +0800 Subject: [PATCH 25/27] fix(#37): stop 403s from logging users out The API client cleared the session on any 403, and the "Publish All" buttons in the Logic and Actions tabs call endpoints that always return 403 PUBLICATION_NOT_ENABLED (publication goes through the ontology publish flow), so clicking them sent the user back to the login page. - Only 401 (after a failed token refresh) logs the user out; a 403 is a permission error, not an expired session. - Remove the two "Publish All" buttons and their orphaned i18n keys. - Hide the admin-only curated approve/reject buttons from non-admins, which used to hit the same 403 path. Closes #37 Co-Authored-By: Claude Opus 5.5 --- frontend/src/api/client.test.ts | 42 +++++++++++++++++++ frontend/src/api/client.ts | 2 +- frontend/src/i18n/en.json | 2 - frontend/src/i18n/zh.json | 2 - .../ontologies/detail/tabs/ActionsTab.tsx | 11 +---- .../pages/ontologies/detail/tabs/LogicTab.tsx | 12 +----- .../pipelines/curated/CuratedTab.test.tsx | 19 +++++++++ .../pages/pipelines/curated/CuratedTab.tsx | 5 ++- 8 files changed, 68 insertions(+), 27 deletions(-) create mode 100644 frontend/src/api/client.test.ts create mode 100644 frontend/src/pages/pipelines/curated/CuratedTab.test.tsx diff --git a/frontend/src/api/client.test.ts b/frontend/src/api/client.test.ts new file mode 100644 index 00000000..2e870e94 --- /dev/null +++ b/frontend/src/api/client.test.ts @@ -0,0 +1,42 @@ +import { afterAll, afterEach, beforeAll, expect, it, vi } from 'vitest' +import { http, HttpResponse } from 'msw' +import { setupServer } from 'msw/node' +import { apiClient, apiClientV2 } from './client' +import { useAuthStore } from '@/stores/authStore' + +const server = setupServer() +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })) +afterAll(() => server.close()) +afterEach(() => { + server.resetHandlers() + vi.restoreAllMocks() + useAuthStore.setState({ token: null, user: null }) +}) + +it.each([['v1', apiClient], ['v2', apiClientV2]] as const)('%s permission denial preserves login without refreshing', async (version, client) => { + useAuthStore.setState({ token: 'valid-session' }) + const fetch = vi.spyOn(globalThis, 'fetch') + server.use(http.get(`*/api/${version}/denied`, () => HttpResponse.json({ detail: 'Forbidden' }, { status: 403 }))) + await expect(client.get('/denied')).rejects.toMatchObject({ detail: 'Forbidden' }) + expect(useAuthStore.getState().token).toBe('valid-session') + expect(fetch).not.toHaveBeenCalled() +}) + +it('refreshes an expired session once and retries with the new token', async () => { + useAuthStore.setState({ token: 'expired' }) + const fetch = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(JSON.stringify({ data: { access_token: 'renewed' } }), { status: 200 })) + server.use(http.get('*/api/v1/protected', ({ request }) => request.headers.get('Authorization') === 'Bearer renewed' + ? HttpResponse.json({ data: { ok: true } }) : HttpResponse.json({}, { status: 401 }))) + await expect(apiClient.get('/protected')).resolves.toEqual({ ok: true }) + expect(useAuthStore.getState().token).toBe('renewed') + expect(fetch).toHaveBeenCalledTimes(1) +}) + +it('clears login if the refreshed token is still rejected', async () => { + useAuthStore.setState({ token: 'expired' }) + const fetch = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(JSON.stringify({ data: { access_token: 'rejected' } }), { status: 200 })) + server.use(http.get('*/api/v1/protected', () => HttpResponse.json({ detail: 'Unauthorized' }, { status: 401 }))) + await expect(apiClient.get('/protected')).rejects.toMatchObject({ detail: 'Unauthorized' }) + expect(useAuthStore.getState().token).toBeNull() + expect(fetch).toHaveBeenCalledTimes(1) +}) diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 2e7b5700..ea6b5e1e 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -65,7 +65,7 @@ function createApiClient(baseURL: string, options?: { runtimeDelegation?: boolea return client(config) } } - if (!options?.runtimeDelegation && !options?.preserveSessionOnAuthorizationFailure && (status === 401 || status === 403)) { + if (!options?.runtimeDelegation && !options?.preserveSessionOnAuthorizationFailure && status === 401) { useAuthStore.getState().logout() } if (options?.runtimeDelegation && status === 401) { diff --git a/frontend/src/i18n/en.json b/frontend/src/i18n/en.json index 50ee752c..4eac8221 100644 --- a/frontend/src/i18n/en.json +++ b/frontend/src/i18n/en.json @@ -247,7 +247,6 @@ "delete_title": "Delete Rule", "delete_confirm": "Delete rule “{{name}}”? This cannot be undone.", "ph_search": "Search rule name / definition…", - "publish_all": "Publish All", "col_name": "Name", "col_definition": "Definition", "col_desc": "Description", @@ -263,7 +262,6 @@ "empty": "No actions yet", "delete_title": "Delete Action", "delete_confirm": "Delete action “{{name}}”? This cannot be undone.", - "publish_all": "Publish All", "col_name": "Name", "col_rule": "Rule", "col_desc": "Description", diff --git a/frontend/src/i18n/zh.json b/frontend/src/i18n/zh.json index 48905891..cd55a126 100644 --- a/frontend/src/i18n/zh.json +++ b/frontend/src/i18n/zh.json @@ -247,7 +247,6 @@ "delete_title": "删除规则", "delete_confirm": "确定删除规则「{{name}}」吗?此操作不可撤销。", "ph_search": "搜索规则名称 / 定义…", - "publish_all": "发布全部", "col_name": "名称", "col_definition": "定义", "col_desc": "描述", @@ -263,7 +262,6 @@ "empty": "暂无动作", "delete_title": "删除动作", "delete_confirm": "确定删除动作「{{name}}」吗?此操作不可撤销。", - "publish_all": "发布全部", "col_name": "名称", "col_rule": "规则", "col_desc": "描述", diff --git a/frontend/src/pages/ontologies/detail/tabs/ActionsTab.tsx b/frontend/src/pages/ontologies/detail/tabs/ActionsTab.tsx index 8a23e603..8b526120 100644 --- a/frontend/src/pages/ontologies/detail/tabs/ActionsTab.tsx +++ b/frontend/src/pages/ontologies/detail/tabs/ActionsTab.tsx @@ -6,7 +6,7 @@ import { useTranslation } from 'react-i18next' import { ontologyApi } from '@/api/ontologies' import { apiClient } from '@/api/client' import ConfirmDialog from '@/components/ConfirmDialog' -import { Pencil, Trash2, Plus, ToggleLeft, ToggleRight, CheckCircle, Loader2 } from 'lucide-react' +import { Pencil, Trash2, Plus, ToggleLeft, ToggleRight } from 'lucide-react' import type { Action } from '@/types/ontology' type ActionRow = Action @@ -36,20 +36,11 @@ export default function ActionsTab({ ontologyId }: { ontologyId: string }) { mutationFn: (id: string) => apiClient.post(`/ontologies/${ontologyId}/actions/${id}/toggle`), onSuccess: () => qc.invalidateQueries({ queryKey: ['actions', ontologyId] }), }) - const publishMut = useMutation({ - mutationFn: () => apiClient.post(`/ontologies/${ontologyId}/actions/publish`), - onSuccess: () => qc.invalidateQueries({ queryKey: ['actions', ontologyId] }), - }) return (
-
-
- {isPending && ( + {canReview && isPending && ( <>
-
diff --git a/frontend/src/types/ontology.ts b/frontend/src/types/ontology.ts index f056b4a3..fd7b4ec9 100644 --- a/frontend/src/types/ontology.ts +++ b/frontend/src/types/ontology.ts @@ -17,6 +17,7 @@ export interface OntologyListItem { export interface OntologyDetail extends OntologyListItem { description?: string build_mode?: string + is_dirty?: boolean created_at: string } From b185154eaa68390c09c65f987cf0d11b5abe9547 Mon Sep 17 00:00:00 2001 From: Jing Wang Date: Fri, 13 Nov 2026 08:35:06 +0800 Subject: [PATCH 27/27] fix(#39): name blocked ontologies and the missing data grant Binding an ontology to an agent requires the read_schema, read_instances and traverse_relations data capabilities whatever tools are selected. These come only from an admin-issued ontology data grant; creating an ontology doesn't grant them. Users without a grant saw a bare "binding conflict" plus an ontology id, and clearing tools changed nothing. - The banner now names the blocked ontology and says to ask an admin for a data grant. - Clear the validation result when the binding set is empty, so an old conflict banner doesn't stay after unbinding. The access model is unchanged: owners still need an explicit data grant. Closes #39 Co-Authored-By: Claude Opus 5.5 --- frontend/src/i18n/en.json | 1 + frontend/src/i18n/zh.json | 1 + .../pages/agents/detail/ToolConfigTab.test.tsx | 16 ++++++++++++++++ .../src/pages/agents/detail/ToolConfigTab.tsx | 12 ++++++++++-- 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/frontend/src/i18n/en.json b/frontend/src/i18n/en.json index 4eac8221..9b6e9412 100644 --- a/frontend/src/i18n/en.json +++ b/frontend/src/i18n/en.json @@ -565,6 +565,7 @@ "tools": { "available_later": "Available later", "bind": "Bind", + "blocked_hint": "Your account has no data grant (read schema, instances and relations) for: {{names}}. Ask an admin to grant data access.", "bound": "Bound", "capabilities": "Capabilities", "external": "External Tools", diff --git a/frontend/src/i18n/zh.json b/frontend/src/i18n/zh.json index cd55a126..29ca657f 100644 --- a/frontend/src/i18n/zh.json +++ b/frontend/src/i18n/zh.json @@ -565,6 +565,7 @@ "tools": { "available_later": "后续提供", "bind": "绑定", + "blocked_hint": "当前账号没有以下本体的数据授权(读取结构、实例和关系):{{names}}。请联系管理员开通数据授权。", "bound": "已绑定", "capabilities": "能力", "external": "外部工具", diff --git a/frontend/src/pages/agents/detail/ToolConfigTab.test.tsx b/frontend/src/pages/agents/detail/ToolConfigTab.test.tsx index f6254cbd..b353db02 100644 --- a/frontend/src/pages/agents/detail/ToolConfigTab.test.tsx +++ b/frontend/src/pages/agents/detail/ToolConfigTab.test.tsx @@ -207,6 +207,22 @@ describe('P2C-TOOLS', () => { expect(bindings[0].enabled_categories).not.toContain('logic') }) + it('clears a stale conflict banner once the ontology is unbound', async () => { + toolsHandlers() + server.use( + http.post('*/api/v1/agents/a-1/tool-validation', () => + HttpResponse.json({ data: { valid: false, blocked: ['o-1'], capabilities: [] }, message: 'ok' })), + ) + renderTab() + await screen.findByTestId('ontology-picker') + await pickOntology('Supply Ontology') + expect(await screen.findByText('绑定配置存在冲突')).toBeTruthy() + // the blocked ontology is named, with the actual fix (a data grant), not a bare id + expect(screen.getByText(/没有以下本体的数据授权.*Supply Ontology.*联系管理员/)).toBeTruthy() + await userEvent.click(screen.getByRole('button', { name: '解绑' })) + await waitFor(() => expect(screen.queryByText('绑定配置存在冲突')).toBeNull()) + }) + it('binds at most one ontology: the picker locks once bound, unbind to switch', async () => { toolsHandlers() server.use( diff --git a/frontend/src/pages/agents/detail/ToolConfigTab.tsx b/frontend/src/pages/agents/detail/ToolConfigTab.tsx index 49ecf906..d7973a48 100644 --- a/frontend/src/pages/agents/detail/ToolConfigTab.tsx +++ b/frontend/src/pages/agents/detail/ToolConfigTab.tsx @@ -147,7 +147,11 @@ export default function ToolConfigTab({ agentId, activeVersion, canEdit, onSaved // validate the proposed binding set whenever it changes useEffect(() => { - if (!canEdit || bindings.length === 0) return + if (!canEdit || bindings.length === 0) { + // drop a stale result so an old conflict banner doesn't outlive the binding + void Promise.resolve().then(() => setValidation(null)) + return + } let cancelled = false agentToolsApi.validateAgentTools(agentId, { ontology_ids: bindings.map(b => b.ontology_id) }) .then(result => { if (!cancelled) setValidation(result) }) @@ -243,7 +247,11 @@ export default function ToolConfigTab({ agentId, activeVersion, canEdit, onSaved : t('agent.tools.invalid', '绑定配置存在冲突')}

{validation.blocked && validation.blocked.length > 0 && ( -

{validation.blocked.join(', ')}

+

+ {t('agent.tools.blocked_hint', { + names: validation.blocked.map(id => ontologies.find(o => o.id === id)?.name ?? id).join(', '), + })} +

)}