diff --git a/.env.example b/.env.example index 4035784b..95db94b5 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,6 @@ MINIO_SECRET_KEY=minioadmin MINIO_USE_SSL=false # v2 — ChromaDB +# Docker network uses port 8000; host-side clients use localhost:8001. CHROMA_HOST=chromadb -CHROMA_PORT=8001 - +CHROMA_PORT=8000 diff --git a/README.md b/README.md index f0a289ce..01d5857a 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ In Ontexus, every ontology is made of these building blocks: | Layer | Technology | |---|---| -| Frontend | React 18, TypeScript, Vite, Tailwind CSS, Cytoscape.js | +| Frontend | React 19, TypeScript, Vite, Tailwind CSS, Cytoscape.js | | Backend | FastAPI, SQLAlchemy, Alembic | | Metadata DB | SQLite (dev) / PostgreSQL (prod) | | Object storage | MinIO (optional, local-file fallback) | diff --git a/backend/app/routers/actions.py b/backend/app/routers/actions.py index a923fc67..cf3bb8fa 100644 --- a/backend/app/routers/actions.py +++ b/backend/app/routers/actions.py @@ -2,6 +2,7 @@ from sqlalchemy.orm import Session from app.deps import get_db, get_current_user, require_editor from app.models.user import User +from app.services.ontology_access import GrantDenied, require_project_grant from app.services.publication.working_copy import OntologyWorkingCopyService from app.services.publication.legacy_sync import delete_action_mirror, sync_action_to_v2 from app.models.action import Action @@ -10,13 +11,22 @@ router = APIRouter() + +def _grant(db: Session, actor: User, ontology_id: str, capability: str) -> dict: + try: + return require_project_grant(db, actor, ontology_id, capability) + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + @router.get("") -def list_actions(ontology_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def list_actions(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") items = db.query(Action).filter(Action.ontology_id == ontology_id).all() return {"data": [ActionOut.model_validate(a).model_dump() for a in items]} @router.post("", status_code=201) def create_action(ontology_id: str, body: ActionCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") def _write(): data = {k: v for k, v in body.model_dump().items() if v is not None} a = Action(id=str(uuid.uuid4()), ontology_id=ontology_id, **data) @@ -29,7 +39,8 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="action.create", callback=_write) @router.get("/{action_id}") -def get_action(ontology_id: str, action_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def get_action(ontology_id: str, action_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") a = db.query(Action).filter(Action.id == action_id, Action.ontology_id == ontology_id).first() if not a: raise HTTPException(404, "Not found") @@ -37,6 +48,7 @@ def get_action(ontology_id: str, action_id: str, db: Session = Depends(get_db), @router.put("/{action_id}") def update_action(ontology_id: str, action_id: str, body: ActionUpdate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") a = db.query(Action).filter(Action.id == action_id, Action.ontology_id == ontology_id).first() if not a: raise HTTPException(404, "Not found") @@ -51,6 +63,7 @@ def _write(): @router.delete("/{action_id}", status_code=204) def delete_action(ontology_id: str, action_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") a = db.query(Action).filter(Action.id == action_id, Action.ontology_id == ontology_id).first() if not a: raise HTTPException(404, "Not found") @@ -62,6 +75,7 @@ def _write(): @router.post("/{action_id}/toggle") def toggle_action(ontology_id: str, action_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") a = db.query(Action).filter(Action.id == action_id, Action.ontology_id == ontology_id).first() if not a: raise HTTPException(404, "Not found") @@ -74,6 +88,7 @@ def _write(): @router.post("/publish") -def publish_actions(ontology_id: str, db: Session = Depends(get_db), _=Depends(require_editor)): +def publish_actions(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): """Direct tool publication is closed; use the Ontology compiler (P1C).""" + _grant(db, current_user, ontology_id, "edit") raise HTTPException(status_code=403, detail="PUBLICATION_NOT_ENABLED") diff --git a/backend/app/routers/audit.py b/backend/app/routers/audit.py index 88246768..5ee5c3ac 100644 --- a/backend/app/routers/audit.py +++ b/backend/app/routers/audit.py @@ -5,13 +5,19 @@ from app.models.ontology import OntologyProject from app.schemas.audit import AuditRequest, AuditTaskOut from app.services.publication.extraction_model_port import default_extraction_model_port +from app.models.user import User +from app.services.ontology_access import GrantDenied, require_project_grant import uuid router = APIRouter() @router.post("") -def start_audit(ontology_id: str, body: AuditRequest, db: Session = Depends(get_db), _=Depends(require_editor)): +def start_audit(ontology_id: str, body: AuditRequest, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + try: + require_project_grant(db, current_user, ontology_id, "edit") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc project = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first() if not project: raise HTTPException(404, "Ontology not found") @@ -50,7 +56,11 @@ def run_sync(): @router.get("/status") -def get_audit_status(ontology_id: str, task_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def get_audit_status(ontology_id: str, task_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + try: + require_project_grant(db, current_user, ontology_id, "read") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc task = db.query(AuditTask).filter(AuditTask.id == task_id, AuditTask.ontology_id == ontology_id).first() if not task: raise HTTPException(404, "Audit task not found") diff --git a/backend/app/routers/entities.py b/backend/app/routers/entities.py index d8651313..d3fa0358 100644 --- a/backend/app/routers/entities.py +++ b/backend/app/routers/entities.py @@ -3,19 +3,29 @@ from app.deps import get_db, get_current_user, require_admin, require_editor from app.models.entity import Entity from app.models.user import User +from app.services.ontology_access import GrantDenied, require_project_grant from app.services.publication.working_copy import OntologyWorkingCopyService from app.schemas.entity import EntityCreate, EntityUpdate, EntityOut import uuid router = APIRouter() + +def _grant(db: Session, actor: User, ontology_id: str, capability: str) -> dict: + try: + return require_project_grant(db, actor, ontology_id, capability) + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + @router.get("") -def list_entities(ontology_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def list_entities(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") items = db.query(Entity).filter(Entity.ontology_id == ontology_id).all() return {"data": [EntityOut.model_validate(e).model_dump() for e in items]} @router.post("", status_code=201) def create_entity(ontology_id: str, body: EntityCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") def _write(): data = {k: v for k, v in body.model_dump().items() if v is not None} e = Entity(id=str(uuid.uuid4()), ontology_id=ontology_id, **data) @@ -24,7 +34,8 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="entity.create", callback=_write) @router.get("/{entity_id}") -def get_entity(ontology_id: str, entity_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def get_entity(ontology_id: str, entity_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") e = db.query(Entity).filter(Entity.id == entity_id, Entity.ontology_id == ontology_id).first() if not e: raise HTTPException(404, "Not found") @@ -32,6 +43,7 @@ def get_entity(ontology_id: str, entity_id: str, db: Session = Depends(get_db), @router.put("/{entity_id}") def update_entity(ontology_id: str, entity_id: str, body: EntityUpdate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") e = db.query(Entity).filter(Entity.id == entity_id, Entity.ontology_id == ontology_id).first() if not e: raise HTTPException(404, "Not found") @@ -44,6 +56,7 @@ def _write(): @router.delete("/{entity_id}", status_code=204) def delete_entity(ontology_id: str, entity_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_admin)): + _grant(db, current_user, ontology_id, "edit") e = db.query(Entity).filter(Entity.id == entity_id, Entity.ontology_id == ontology_id).first() if not e: raise HTTPException(404, "Not found") @@ -56,9 +69,10 @@ def list_entity_instances( ontology_id: str, entity_id: str, db: Session = Depends(get_db), - _=Depends(get_current_user), + current_user: User = Depends(get_current_user), ): """概念实体下挂的行级实例数据(Pipeline Mapping 与简易 LLM 提取共用同一张表)""" + _grant(db, current_user, ontology_id, "read") from app.models.entity_instance import EntityInstance entity = db.query(Entity).filter( diff --git a/backend/app/routers/export.py b/backend/app/routers/export.py index 8604fdc2..ae6078e0 100644 --- a/backend/app/routers/export.py +++ b/backend/app/routers/export.py @@ -3,6 +3,7 @@ from sqlalchemy.orm import Session from app.deps import get_db, get_current_user from app.services import export_service +from app.services.ontology_access import GrantDenied, require_project_grant router = APIRouter() @@ -21,8 +22,12 @@ def export_ontology( ontology_id: str, format: str = Query("json", pattern="^(json|yaml|csv|ttl|html|cypher|tugraph)$"), db: Session = Depends(get_db), - _=Depends(get_current_user) + current_user=Depends(get_current_user) ): + try: + require_project_grant(db, current_user, ontology_id, "read") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc if format not in FORMAT_MAP: raise HTTPException(400, f"Unsupported format: {format}") content_type, ext, fn = FORMAT_MAP[format] diff --git a/backend/app/routers/extraction.py b/backend/app/routers/extraction.py index 81a27ec2..6d9828d6 100644 --- a/backend/app/routers/extraction.py +++ b/backend/app/routers/extraction.py @@ -9,6 +9,7 @@ from app.models.ontology import OntologyProject from app.services.document_service import combine_converted_files from app.schemas.extraction import ExtractionRequest, ExtractionTaskOut +from app.services.ontology_access import GrantDenied, require_project_grant import uuid logger = logging.getLogger(__name__) @@ -16,7 +17,11 @@ router = APIRouter() @router.post("") -def start_extraction(ontology_id: str, body: ExtractionRequest, db: Session = Depends(get_db), _=Depends(require_editor)): +def start_extraction(ontology_id: str, body: ExtractionRequest, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + try: + require_project_grant(db, current_user, ontology_id, "edit") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc project = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first() if not project: raise HTTPException(404, "Ontology not found") @@ -67,7 +72,11 @@ def run_sync(): return {"data": {"task_id": task.id}, "message": "Extraction queued"} @router.get("/status") -def get_extraction_status(ontology_id: str, task_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def get_extraction_status(ontology_id: str, task_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + try: + require_project_grant(db, current_user, ontology_id, "read") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc task = db.query(ExtractionTask).filter(ExtractionTask.id == task_id, ExtractionTask.ontology_id == ontology_id).first() if not task: raise HTTPException(404, "Task not found") diff --git a/backend/app/routers/files.py b/backend/app/routers/files.py index 926e1d14..52f7480c 100644 --- a/backend/app/routers/files.py +++ b/backend/app/routers/files.py @@ -1,5 +1,6 @@ import os import uuid +import logging from fastapi import APIRouter, Depends, HTTPException, UploadFile, File from sqlalchemy.orm import Session from app.deps import get_db, get_current_user, require_admin, require_editor @@ -7,9 +8,11 @@ from app.models.ontology import OntologyProject from app.schemas.file import FileOut from app.services.document_service import convert_document, is_usable_converted_text +from app.services.ontology_access import GrantDenied, require_project_grant from app.config import settings router = APIRouter() +logger = logging.getLogger(__name__) ALLOWED_TYPES = { "application/pdf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", @@ -39,8 +42,16 @@ def _file_out(f: UploadedFile) -> dict: ).model_dump() +def _grant(db: Session, actor, ontology_id: str, capability: str) -> dict: + try: + return require_project_grant(db, actor, ontology_id, capability) + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + + @router.get("") -def list_files(ontology_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def list_files(ontology_id: str, db: Session = Depends(get_db), current_user=Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") files = db.query(UploadedFile).filter(UploadedFile.ontology_id == ontology_id).all() return {"data": [_file_out(f) for f in files]} @@ -49,11 +60,12 @@ async def upload_file( ontology_id: str, file: UploadFile = File(...), db: Session = Depends(get_db), - _=Depends(require_editor) + current_user=Depends(require_editor), ): + _grant(db, current_user, ontology_id, "edit") project = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first() if not project: - raise HTTPException(404, "Ontology not found") + raise HTTPException(404, detail="ONTOLOGY_NOT_FOUND") ext_name = (file.filename or "").rsplit(".", 1)[-1].lower() allowed = {e.strip() for e in settings.allowed_upload_extensions.split(",") if e.strip()} @@ -67,36 +79,63 @@ async def upload_file( ext = os.path.splitext(file.filename)[1] save_path = os.path.join(upload_dir, f"{file_id}{ext}") - content = await file.read() - if len(content) > settings.max_upload_mb * 1024 * 1024: + max_bytes = settings.max_upload_mb * 1024 * 1024 + declared_size = getattr(file, "size", None) + if declared_size is not None and declared_size > max_bytes: raise HTTPException(413, f"文件超过大小限制 {settings.max_upload_mb}MB") - with open(save_path, "wb") as f: - f.write(content) - - mime = file.content_type or "application/octet-stream" - conversion = convert_document(save_path, mime) - if not conversion.ok: - if os.path.exists(save_path): - os.remove(save_path) - raise HTTPException(422, conversion.error or "文件转换失败") - - db_file = UploadedFile( - id=file_id, - ontology_id=ontology_id, - filename=file.filename, - file_path=save_path, - file_size=len(content), - mime_type=mime, - converted_md=conversion.content, - ) - db.add(db_file); db.commit(); db.refresh(db_file) - return {"data": _file_out(db_file)} + + committed = False + try: + total_bytes = 0 + with open(save_path, "wb") as saved: + while True: + chunk = await file.read(1024 * 1024) + if not chunk: + break + total_bytes += len(chunk) + if total_bytes > max_bytes: + raise HTTPException(413, f"文件超过大小限制 {settings.max_upload_mb}MB") + saved.write(chunk) + + mime = file.content_type or "application/octet-stream" + conversion = convert_document(save_path, mime) + if not conversion.ok: + raise HTTPException(422, conversion.error or "文件转换失败") + + db_file = UploadedFile( + id=file_id, + ontology_id=ontology_id, + filename=file.filename, + file_path=save_path, + file_size=total_bytes, + mime_type=mime, + converted_md=conversion.content, + ) + db.add(db_file) + try: + db.commit() + except Exception: + db.rollback() + raise + committed = True + db.refresh(db_file) + return {"data": _file_out(db_file)} + except Exception: + if not committed and os.path.exists(save_path): + try: + os.remove(save_path) + except OSError: + logger.warning("failed to clean up rejected upload %s", save_path, exc_info=True) + raise @router.delete("/{file_id}", status_code=204) -def delete_file(ontology_id: str, file_id: str, db: Session = Depends(get_db), _=Depends(require_admin)): +def delete_file(ontology_id: str, file_id: str, db: Session = Depends(get_db), current_user=Depends(require_admin)): + _grant(db, current_user, ontology_id, "edit") f = db.query(UploadedFile).filter(UploadedFile.id == file_id, UploadedFile.ontology_id == ontology_id).first() if not f: raise HTTPException(404, "File not found") - if os.path.exists(f.file_path): - os.remove(f.file_path) - db.delete(f); db.commit() + file_path = f.file_path + db.delete(f) + db.commit() + if os.path.exists(file_path): + os.remove(file_path) diff --git a/backend/app/routers/graph.py b/backend/app/routers/graph.py index ea6bca00..61aeee57 100644 --- a/backend/app/routers/graph.py +++ b/backend/app/routers/graph.py @@ -5,13 +5,22 @@ from app.models.relation import Relation from app.models.ontology import OntologyProject from app.models.user import User +from app.services.ontology_access import GrantDenied, require_project_grant from app.services.publication.working_copy import OntologyWorkingCopyService from app.services.relation_dedup import dedupe_relation_rows router = APIRouter() + +def _grant(db: Session, actor: User, ontology_id: str, capability: str) -> dict: + try: + return require_project_grant(db, actor, ontology_id, capability) + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + @router.get("") -def get_graph(ontology_id: str, limit: int = 300, db: Session = Depends(get_db), _=Depends(get_current_user)): +def get_graph(ontology_id: str, limit: int = 300, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") project = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first() if not project: raise HTTPException(404, "Ontology not found") @@ -138,12 +147,26 @@ def create_relation( ): from app.models.relation import Relation import uuid + _grant(db, current_user, ontology_id, "edit") + source_entity = body.get("source_entity") + target_entity = body.get("target_entity") + if not source_entity or not target_entity: + raise HTTPException(status_code=422, detail="source_entity and target_entity are required") + source = db.query(Entity).filter( + Entity.id == source_entity, Entity.ontology_id == ontology_id + ).first() + target = db.query(Entity).filter( + Entity.id == target_entity, Entity.ontology_id == ontology_id + ).first() + if source is None or target is None: + raise HTTPException(status_code=422, detail="RELATION_ENTITIES_MUST_BELONG_TO_ONTOLOGY") + def _write(): relation = Relation( id=str(uuid.uuid4()), ontology_id=ontology_id, - source_entity=body["source_entity"], - target_entity=body["target_entity"], + source_entity=source_entity, + target_entity=target_entity, type=body.get("type", "关联"), properties=body.get("properties", {}), confidence=body.get("confidence", 1.0), @@ -154,6 +177,7 @@ def _write(): @router.delete("/relations/{relation_id}", status_code=204) def delete_relation(ontology_id: str, relation_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_admin)): + _grant(db, current_user, ontology_id, "edit") r = db.query(Relation).filter(Relation.id == relation_id, Relation.ontology_id == ontology_id).first() if not r: raise HTTPException(404, "Not found") diff --git a/backend/app/routers/logic.py b/backend/app/routers/logic.py index f2e9b51b..8b3e867b 100644 --- a/backend/app/routers/logic.py +++ b/backend/app/routers/logic.py @@ -2,6 +2,7 @@ from sqlalchemy.orm import Session from app.deps import get_db, get_current_user, require_editor from app.models.user import User +from app.services.ontology_access import GrantDenied, require_project_grant from app.services.publication.working_copy import OntologyWorkingCopyService from app.services.publication.legacy_sync import delete_logic_rule_mirror, sync_logic_rule_to_v2 from app.models.logic import LogicRule @@ -10,13 +11,22 @@ router = APIRouter() + +def _grant(db: Session, actor: User, ontology_id: str, capability: str) -> dict: + try: + return require_project_grant(db, actor, ontology_id, capability) + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + @router.get("") -def list_logic(ontology_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def list_logic(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") items = db.query(LogicRule).filter(LogicRule.ontology_id == ontology_id).all() return {"data": [LogicRuleOut.model_validate(r).model_dump() for r in items]} @router.post("", status_code=201) def create_logic(ontology_id: str, body: LogicRuleCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") def _write(): data = {k: v for k, v in body.model_dump().items() if v is not None} r = LogicRule(id=str(uuid.uuid4()), ontology_id=ontology_id, **data) @@ -29,7 +39,8 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="logic.create", callback=_write) @router.get("/{logic_id}") -def get_logic(ontology_id: str, logic_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def get_logic(ontology_id: str, logic_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") r = db.query(LogicRule).filter(LogicRule.id == logic_id, LogicRule.ontology_id == ontology_id).first() if not r: raise HTTPException(404, "Not found") @@ -37,6 +48,7 @@ def get_logic(ontology_id: str, logic_id: str, db: Session = Depends(get_db), _= @router.put("/{logic_id}") def update_logic(ontology_id: str, logic_id: str, body: LogicRuleUpdate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") r = db.query(LogicRule).filter(LogicRule.id == logic_id, LogicRule.ontology_id == ontology_id).first() if not r: raise HTTPException(404, "Not found") @@ -51,6 +63,7 @@ def _write(): @router.delete("/{logic_id}", status_code=204) def delete_logic(ontology_id: str, logic_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") r = db.query(LogicRule).filter(LogicRule.id == logic_id, LogicRule.ontology_id == ontology_id).first() if not r: raise HTTPException(404, "Not found") @@ -63,6 +76,7 @@ def _write(): @router.post("/{logic_id}/toggle") def toggle_logic(ontology_id: str, logic_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): """Human Review: 启用/禁用规则""" + _grant(db, current_user, ontology_id, "edit") r = db.query(LogicRule).filter(LogicRule.id == logic_id, LogicRule.ontology_id == ontology_id).first() if not r: raise HTTPException(404, "Not found") @@ -75,6 +89,7 @@ def _write(): @router.post("/publish") -def publish_logic_rules(ontology_id: str, db: Session = Depends(get_db), _=Depends(require_editor)): +def publish_logic_rules(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): """Human Review: 发布所有草稿规则 — direct tool publication is closed; use the Ontology compiler (P1C).""" + _grant(db, current_user, ontology_id, "edit") raise HTTPException(status_code=403, detail="PUBLICATION_NOT_ENABLED") diff --git a/backend/app/routers/models.py b/backend/app/routers/models.py index 58227054..9b68eac2 100644 --- a/backend/app/routers/models.py +++ b/backend/app/routers/models.py @@ -39,6 +39,7 @@ remediate_blocked_identity, rotate_credential, select_active_version, + versioning_schema_present, ) router = APIRouter() @@ -325,14 +326,32 @@ def delete_model(model_id: str, db: Session = Depends(get_db), _=Depends(require c = db.query(ModelConfig).filter(ModelConfig.id == model_id).first() if not c: raise HTTPException(404, "Not found") + # Block only on a REAL dependent — an Agent version pinned to one of this + # model's versions, or a prompt-generation provenance row — not merely on + # the model's own version-1 row that create_model() always bootstraps. + # Checking "any model_config_versions row exists" made every model saved + # through the normal create flow permanently undeletable. referenced = db.execute(text( - "SELECT count(*) FROM model_config_versions WHERE model_config_id = :id" + "SELECT count(*) FROM model_config_versions v WHERE v.model_config_id = :id AND (" + "EXISTS (SELECT 1 FROM agent_versions av WHERE av.default_model_config_version_id = v.id) " + "OR EXISTS (SELECT 1 FROM prompt_generations pg WHERE pg.model_config_version_id = v.id))" ), {"id": model_id}).scalar_one() if referenced > 0: raise HTTPException(409, detail="MODEL_REFERENCED") db.query(ExtractionTask).filter(ExtractionTask.model_id == model_id).update( {ExtractionTask.model_id: None}, synchronize_session=False ) + # model_config_versions/model_credentials/model_migration_findings all + # RESTRICT-reference model_configs.id, so they must be cleared first — + # and model_configs.active_version_id RESTRICT-references the version + # row itself (the reverse direction), so that pointer must be nulled + # before the version row can go. The pointer is migration-only; the + # legacy metadata.create_all schema has no such column. + if versioning_schema_present(db): + db.execute(text("UPDATE model_configs SET active_version_id = NULL WHERE id = :id"), {"id": model_id}) + db.execute(text("DELETE FROM model_credentials WHERE model_config_id = :id"), {"id": model_id}) + db.execute(text("DELETE FROM model_migration_findings WHERE model_config_id = :id"), {"id": model_id}) + db.execute(text("DELETE FROM model_config_versions WHERE model_config_id = :id"), {"id": model_id}) db.delete(c) db.commit() diff --git a/backend/app/routers/ontologies.py b/backend/app/routers/ontologies.py index 143d9b38..62ef6360 100644 --- a/backend/app/routers/ontologies.py +++ b/backend/app/routers/ontologies.py @@ -1,3 +1,5 @@ +import logging + from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy.orm import Session from sqlalchemy import func, text @@ -9,19 +11,34 @@ from app.models.relation import Relation from app.models.user import User from app.schemas.ontology import OntologyCreate, OntologyOut, OntologyListItem, OntologyUpdate -from app.services.ontology_access import creator_grant +from app.services.ontology_access import ( + GrantDenied, + creator_grant, + discoverable_ontology_ids, + require_project_grant, +) from app.services.publication.working_copy import OntologyWorkingCopyService import uuid +logger = logging.getLogger(__name__) router = APIRouter() + +def _grant(db: Session, actor: User, ontology_id: str, capability: str) -> dict: + try: + return require_project_grant(db, actor, ontology_id, capability) + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + @router.get("") def list_ontologies( name: Optional[str] = None, page: int = 1, page_size: int = 20, - db: Session = Depends(get_db), _=Depends(get_current_user) + db: Session = Depends(get_db), current_user: User = Depends(get_current_user) ): - q = db.query(OntologyProject) + q = db.query(OntologyProject).filter( + OntologyProject.id.in_(discoverable_ontology_ids(db, current_user)) + ) if name: q = q.filter(OntologyProject.name.ilike(f"%{name}%")) total = q.count() @@ -41,7 +58,8 @@ def create_ontology(body: OntologyCreate, db: Session = Depends(get_db), current raise HTTPException(status_code=409, detail={"error": "DUPLICATE_NAME", "message": f"Ontology 名称「{body.name}」已存在", "existing_id": existing.id}) project = OntologyProject(id=str(uuid.uuid4()), name=body.name, domain=body.domain, description=body.description, build_mode=body.build_mode or "simple_llm", - created_by=current_user.id) + created_by=current_user.id, + security_domain_id=current_user.security_domain_id) db.add(project); db.flush() # atomically insert the exact creator grant (P1A-ACCESS contract) creator_grant(db, project.id, current_user) @@ -49,17 +67,19 @@ def create_ontology(body: OntologyCreate, db: Session = Depends(get_db), current return {"data": OntologyOut.model_validate(project).model_dump()} @router.get("/{ontology_id}") -def get_ontology(ontology_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)): +def get_ontology(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): + _grant(db, current_user, ontology_id, "read") p = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first() if not p: - raise HTTPException(404, "Not found") + raise HTTPException(404, detail="ONTOLOGY_NOT_FOUND") return {"data": OntologyOut.model_validate(p).model_dump()} @router.put("/{ontology_id}") def update_ontology(ontology_id: str, body: OntologyUpdate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): + _grant(db, current_user, ontology_id, "edit") p = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first() if not p: - raise HTTPException(404, "Not found") + raise HTTPException(404, detail="ONTOLOGY_NOT_FOUND") def _write(): for k, v in body.model_dump(exclude_none=True).items(): setattr(p, k, v) @@ -68,10 +88,11 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="ontology.update", callback=_write) @router.delete("/{ontology_id}", status_code=204) -def delete_ontology(ontology_id: str, db: Session = Depends(get_db), _=Depends(require_admin)): +def delete_ontology(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_admin)): + _grant(db, current_user, ontology_id, "edit") p = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first() if not p: - raise HTTPException(404, "Not found") + raise HTTPException(404, detail="ONTOLOGY_NOT_FOUND") bound_agent = db.execute(text( "SELECT 1 FROM agent_ontology_bindings WHERE ontology_id = :id LIMIT 1" ), {"id": ontology_id}).scalar_one_or_none() @@ -82,9 +103,14 @@ def delete_ontology(ontology_id: str, db: Session = Depends(get_db), _=Depends(r # gone, unlike execution history (agent runs, sandbox simulations, # business-journey records, MCP write requests) rooted through # ontology_releases, which a delete must not silently wipe. + # entity_instance_relations (and entity_instances, cascade-deleted below) + # carry an AFTER DELETE trigger that re-emits agent_index_outbox rows — + # agent_index_outbox must be cleaned LAST so it sweeps up anything those + # triggers just recreated, or the delete below hits ONTOLOGY_HAS_DEPENDENT_DATA + # on an ontology that still has real instance data. for table in ( "ontology_data_grants", "ontology_project_access_grants", "ontology_migration_findings", - "agent_index_outbox", "entity_property_definitions", "entity_instance_relations", + "entity_property_definitions", "entity_instance_relations", "agent_index_outbox", ): db.execute(text(f"DELETE FROM {table} WHERE ontology_id = :id"), {"id": ontology_id}) try: @@ -93,3 +119,18 @@ def delete_ontology(ontology_id: str, db: Session = Depends(get_db), _=Depends(r except IntegrityError: db.rollback() raise HTTPException(409, detail="ONTOLOGY_HAS_DEPENDENT_DATA") + + # Best-effort external-store cleanup — never fails an already-committed + # delete; Neo4j/Chroma hold per-ontology graph/vector data with no other + # owner once the Postgres row is gone (mirrors the try/except-and-log + # pattern in app/tasks/extraction.py's _sync_neo4j). + try: + from app.services.v2.graph.neo4j_service import Neo4jService + Neo4jService().delete_by_ontology(ontology_id) + except Exception: + logger.warning("Neo4j cleanup failed for deleted ontology %s", ontology_id, exc_info=True) + try: + from app.services.v2.vector.chroma_service import ChromaService + ChromaService().delete_collection(f"ontology_{ontology_id}") + except Exception: + logger.warning("Chroma cleanup failed for deleted ontology %s", ontology_id, exc_info=True) diff --git a/backend/app/routers/overview.py b/backend/app/routers/overview.py index 18247dc6..420f4dba 100644 --- a/backend/app/routers/overview.py +++ b/backend/app/routers/overview.py @@ -3,23 +3,29 @@ from sqlalchemy import func from app.deps import get_db, get_current_user from app.models.ontology import OntologyProject +from app.models.user import User +from app.services.ontology_access import discoverable_ontology_ids router = APIRouter() -def _safe_count(db, model): +def _safe_count(db, model, ontology_ids): try: - return db.query(model).count() + return db.query(model).filter(model.ontology_id.in_(ontology_ids)).count() except Exception: return 0 @router.get("/stats") -def get_stats(db: Session = Depends(get_db), _=Depends(get_current_user)): +def get_stats(db: Session = Depends(get_db), current_user: User = Depends(get_current_user)): from app.models.entity import Entity from app.models.logic import LogicRule from app.models.action import Action + discoverable_ids = discoverable_ontology_ids(db, current_user) + # Recent ontologies - recent = db.query(OntologyProject).order_by(OntologyProject.updated_at.desc()).limit(6).all() + recent = db.query(OntologyProject).filter( + OntologyProject.id.in_(discoverable_ids) + ).order_by(OntologyProject.updated_at.desc()).limit(6).all() recent_list = [] for o in recent: entity_count = db.query(Entity).filter(Entity.ontology_id == o.id).count() @@ -39,6 +45,7 @@ def get_stats(db: Session = Depends(get_db), _=Depends(get_current_user)): # Domain distribution domain_rows = ( db.query(OntologyProject.domain, func.count(OntologyProject.id)) + .filter(OntologyProject.id.in_(discoverable_ids)) .group_by(OntologyProject.domain) .all() ) @@ -47,6 +54,7 @@ def get_stats(db: Session = Depends(get_db), _=Depends(get_current_user)): # Status breakdown status_rows = ( db.query(OntologyProject.status, func.count(OntologyProject.id)) + .filter(OntologyProject.id.in_(discoverable_ids)) .group_by(OntologyProject.status) .all() ) @@ -54,10 +62,10 @@ def get_stats(db: Session = Depends(get_db), _=Depends(get_current_user)): return { "data": { - "ontology_count": _safe_count(db, OntologyProject), - "entity_count": _safe_count(db, Entity), - "logic_count": _safe_count(db, LogicRule), - "action_count": _safe_count(db, Action), + "ontology_count": db.query(OntologyProject).filter(OntologyProject.id.in_(discoverable_ids)).count(), + "entity_count": _safe_count(db, Entity, discoverable_ids), + "logic_count": _safe_count(db, LogicRule, discoverable_ids), + "action_count": _safe_count(db, Action, discoverable_ids), "recent_ontologies": recent_list, "domain_counts": domain_counts, "status_counts": status_counts, diff --git a/backend/app/routers/v2/connections.py b/backend/app/routers/v2/connections.py index 5645581c..36e30900 100644 --- a/backend/app/routers/v2/connections.py +++ b/backend/app/routers/v2/connections.py @@ -11,7 +11,7 @@ import json from datetime import datetime, timezone -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel from sqlalchemy.orm import Session from typing import Optional @@ -19,6 +19,12 @@ from app.database import SessionLocal from app.deps import get_current_user, require_editor from app.models.v2.connection import Connection +from app.models.user import User +from app.services.connection.config import ( + ConnectionConfigError, + build_connector_config, + safe_connection_error, +) from app.services.connection.registry import get_connector router = APIRouter(dependencies=[Depends(get_current_user)]) @@ -53,7 +59,7 @@ class Config: # ── 端点 ────────────────────────────────────────────────────── @router.post("", response_model=ConnectionResponse, status_code=201) -def create_connection(body: ConnectionCreate, db: Session = Depends(get_db), _=Depends(require_editor)): +def create_connection(body: ConnectionCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): """创建连接。config 加密后存储。""" from app.services import encryption_service encrypted_config = {"_encrypted": encryption_service.encrypt(json.dumps(body.config))} @@ -63,6 +69,7 @@ def create_connection(body: ConnectionCreate, db: Session = Depends(get_db), _=D kind=body.kind, config=encrypted_config, status="inactive", + created_by=current_user.id, ) db.add(conn) db.commit() @@ -94,35 +101,36 @@ def _build_db_config(raw_config: dict, db_type: str) -> dict: connection_string,这里组装成 SQLAlchemy 可用的连接 URL。 密码中的特殊字符通过 urllib.parse.quote 编码以避免 URL 解析歧义。 """ - from urllib.parse import quote - host = raw_config.get("host", "localhost") - port = raw_config.get("port", "3306" if db_type == "mysql" else "5432") - user = raw_config.get("user", "") - password = raw_config.get("password", "") - database = raw_config.get("database", "") - # 密码/用户名/库名中的特殊字符(如 @ : / # 空格等)必须 URL 编码, - # 否则 SQLAlchemy 的 URL 解析器会将 @ 等视为 URL 结构分隔符而非密码的一部分。 - # host 不编码(IPv6 地址用 [] 括起,需原样保留)。 - scheme = "mysql+pymysql" if db_type == "mysql" else "postgresql" - conn_str = f"{scheme}://{quote(user, safe='')}:{quote(password, safe='')}@{host}:{port}/{quote(database, safe='')}" - config = dict(raw_config) - config["connection_string"] = conn_str - return config + return build_connector_config(db_type, raw_config) + + +def _load_connection_connector( + conn: Connection, + *, + resource: str | None = None, +): + """Build a connector from an encrypted Connection row. + + Keeping this in one helper ensures API tests, pipeline execution, and + refresh workers all resolve the same saved database configuration. + """ + config = build_connector_config(conn.kind, conn.config, resource=resource) + return get_connector(conn.kind, config) @router.post("/test-config") def test_connection_config(body: TestConfigBody, _=Depends(require_editor)): """测试连接配置(无需先创建 Connection,供 Builder 使用)""" try: - cfg = body.config - if body.type in ("mysql", "postgresql", "postgres"): - db_type = "postgres" if "postgres" in body.type else "mysql" - cfg = _build_db_config(cfg, db_type) + cfg = build_connector_config(body.type, body.config) connector = get_connector(body.type, cfg) ok = connector.test_connection() - return {"success": ok} + result = {"success": ok} + if not ok: + result["detail"] = getattr(connector, "last_error", None) or "connection test failed" + return result except Exception as e: - return {"success": False, "detail": str(e)} + return {"success": False, "detail": safe_connection_error(e)} @router.post("/{connection_id}/test") @@ -132,27 +140,67 @@ def test_connection(connection_id: str, db: Session = Depends(get_db), _=Depends if not conn: raise HTTPException(status_code=404, detail="Connection not found") - from app.services import encryption_service - raw = conn.config.get("_encrypted", "") try: - config = json.loads(encryption_service.decrypt(raw)) if raw else conn.config - except Exception: - config = conn.config - - # 与 test-config 一样,ConnectionsTab 存储的是独立字段而非 connection_string - if conn.kind in ("mysql", "postgres"): - config = _build_db_config(config, conn.kind) - - try: - connector = get_connector(conn.kind, config) + connector = _load_connection_connector(conn) ok = connector.test_connection() conn.status = "active" if ok else "error" db.commit() - return {"success": ok, "status": conn.status} + result = {"success": ok, "status": conn.status} + if not ok: + result["detail"] = getattr(connector, "last_error", None) or "connection test failed" + return result except Exception as e: conn.status = "error" db.commit() - return {"success": False, "status": "error", "detail": str(e)} + return {"success": False, "status": "error", "detail": safe_connection_error(e)} + + +@router.get("/{connection_id}/resources") +def list_connection_resources( + connection_id: str, + db: Session = Depends(get_db), + _=Depends(require_editor), +): + """List tables/collections exposed by a saved connection.""" + conn = db.query(Connection).filter(Connection.id == connection_id).first() + if not conn: + raise HTTPException(404, "Connection not found") + try: + connector = _load_connection_connector(conn) + resources = connector.list_resources() + return {"connection_id": connection_id, "resources": resources} + except (ConnectionConfigError, ValueError) as exc: + raise HTTPException(422, detail=str(exc)) from exc + except Exception as exc: + raise HTTPException( + 502, + detail=f"Unable to list resources for connection '{conn.name}': {safe_connection_error(exc)}", + ) from exc + + +@router.get("/{connection_id}/resources/{resource:path}/sample") +def sample_connection_resource( + connection_id: str, + resource: str, + limit: int = Query(20, ge=1, le=1000), + db: Session = Depends(get_db), + _=Depends(require_editor), +): + """Read a bounded sample from one saved connection resource.""" + conn = db.query(Connection).filter(Connection.id == connection_id).first() + if not conn: + raise HTTPException(404, "Connection not found") + try: + connector = _load_connection_connector(conn, resource=resource) + rows = connector.pull_sample(resource, limit=limit) + return {"connection_id": connection_id, "resource": resource, "rows": rows} + except (ConnectionConfigError, ValueError) as exc: + raise HTTPException(422, detail=str(exc)) from exc + except Exception as exc: + raise HTTPException( + 502, + detail=f"Unable to read resource '{resource}': {safe_connection_error(exc)}", + ) from exc class RefreshConfigurationUpdate(BaseModel): diff --git a/backend/app/routers/v2/curated.py b/backend/app/routers/v2/curated.py index 53ee2cba..2eabe4fe 100644 --- a/backend/app/routers/v2/curated.py +++ b/backend/app/routers/v2/curated.py @@ -79,14 +79,34 @@ def delete_curated(dataset_id: str, db: Session = Depends(get_db), _admin=Depend raise HTTPException(404, "Curated dataset not found") # 清理关联版本和媒体项 ver_ids = [v.id for v in db.query(DatasetVersion).filter(DatasetVersion.dataset_id == dataset_id).all()] + media_uris: list[str] = [] if ver_ids: + media_items = db.query(MediaItem).filter(MediaItem.dataset_version_id.in_(ver_ids)).all() + for m in media_items: + media_uris.append(m.storage_uri) + if m.ocr_result_uri: + media_uris.append(m.ocr_result_uri) db.query(MediaItem).filter(MediaItem.dataset_version_id.in_(ver_ids)).delete(synchronize_session=False) db.query(DatasetVersion).filter(DatasetVersion.dataset_id == dataset_id).delete(synchronize_session=False) # 清理审核记录 db.query(CuratedReview).filter(CuratedReview.curated_dataset_id == dataset_id).delete(synchronize_session=False) + # 若曾发起过审核,_ensure_curated_dataset_row/旧版 /review 端点会用相同 id + # 懒创建一条 v2_curated_datasets 镜像行作为 CuratedReview 的 FK 桥接——上面 + # 只删了 v2_datasets 里的真本体行,这条镜像行没有别的清理入口,不删就永久残留 + db.query(CuratedDataset).filter(CuratedDataset.id == dataset_id).delete(synchronize_session=False) db.delete(ds) db.commit() + # Best-effort MinIO cleanup — never fails an already-committed delete + if media_uris: + from app.services.storage_service import get_storage_service + storage = get_storage_service() + for uri in media_uris: + try: + storage.delete_object(uri) + except Exception: + logger.warning("MinIO cleanup failed for object %s (dataset %s)", uri, dataset_id, exc_info=True) + @router.get("/{dataset_id}", response_model=CuratedDatasetResponse) def get_curated(dataset_id: str, db: Session = Depends(get_db)): diff --git a/backend/app/routers/v2/datasets.py b/backend/app/routers/v2/datasets.py index f1811abc..bbc4d273 100644 --- a/backend/app/routers/v2/datasets.py +++ b/backend/app/routers/v2/datasets.py @@ -23,6 +23,30 @@ class DatasetResponse(BaseModel): class Config: from_attributes = True + +_UPLOAD_READ_CHUNK_SIZE = 1024 * 1024 + + +async def _read_upload_bounded( + file: UploadFile, + max_bytes: int, + *, + chunk_size: int = _UPLOAD_READ_CHUNK_SIZE, + error_detail: str | None = None, +) -> bytes: + """Read at most one byte beyond the configured upload limit.""" + content = bytearray() + while len(content) <= max_bytes: + read_size = min(chunk_size, max_bytes - len(content) + 1) + chunk = await file.read(read_size) + if not chunk: + break + if len(content) + len(chunk) > max_bytes: + raise HTTPException(status_code=413, detail=error_detail or "文件超过大小限制") + content.extend(chunk) + return bytes(content) + + @router.post("/upload", status_code=201) async def upload_dataset(file: UploadFile = File(...), db: Session = Depends(get_db), _=Depends(require_editor)): """上传 CSV/Excel 文件,自动创建 raw Dataset + DatasetVersion""" @@ -35,9 +59,12 @@ async def upload_dataset(file: UploadFile = File(...), db: Session = Depends(get if ext not in allowed: raise HTTPException(400, f"不支持的文件类型: .{ext} (允许: {settings.allowed_upload_extensions})") - content = await file.read() - if len(content) > settings.max_upload_mb * 1024 * 1024: - raise HTTPException(413, f"文件超过大小限制 {settings.max_upload_mb}MB") + max_upload_bytes = settings.max_upload_mb * 1024 * 1024 + content = await _read_upload_bounded( + file, + max_upload_bytes, + error_detail=f"文件超过大小限制 {settings.max_upload_mb}MB", + ) # 推断 kind if ext in ("csv", "xlsx", "xls"): kind = "structured" diff --git a/backend/app/routers/v2/graph.py b/backend/app/routers/v2/graph.py index 7294aafe..c173f9c0 100644 --- a/backend/app/routers/v2/graph.py +++ b/backend/app/routers/v2/graph.py @@ -7,6 +7,8 @@ from app.services.publication.working_copy import OntologyWorkingCopyService from app.services.relation_dedup import dedupe_relation_edges as _dedupe_relation_edges from app.database import SessionLocal +from sqlalchemy.orm import Session +from app.services.ontology_access import GrantDenied, require_project_grant router = APIRouter(dependencies=[Depends(get_current_user)]) @@ -24,12 +26,26 @@ def get_db(): db.close() +def _require_read(ontology_id: str, db: Session = Depends(get_db), actor: User = Depends(get_current_user)): + try: + require_project_grant(db, actor, ontology_id, "read") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + + +def _require_edit(ontology_id: str, db: Session = Depends(get_db), actor: User = Depends(require_editor)): + try: + require_project_grant(db, actor, ontology_id, "edit") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + + class CypherRequest(BaseModel): query: str params: dict = {} -@router.get("/{ontology_id}/graph") +@router.get("/{ontology_id}/graph", dependencies=[Depends(_require_read)]) def get_graph(ontology_id: str, limit: int = 200, label_filter: str | None = None): """返回本体图谱数据 (Neovis.js 兼容格式)""" svc = get_neo4j() @@ -95,7 +111,7 @@ def _sqlite_graph_data(ontology_id: str, limit: int = 200, label_filter: str | N db.close() -@router.get("/{ontology_id}/graph/quality") +@router.get("/{ontology_id}/graph/quality", dependencies=[Depends(_require_read)]) def graph_quality(ontology_id: str): from app.models.entity import Entity from app.models.relation import Relation @@ -146,7 +162,7 @@ def graph_quality(ontology_id: str): db.close() -@router.get("/{ontology_id}/integrations/status") +@router.get("/{ontology_id}/integrations/status", dependencies=[Depends(_require_read)]) def integration_status(ontology_id: str): neo = get_neo4j() neo_available = neo.available @@ -161,7 +177,7 @@ def integration_status(ontology_id: str): } -@router.post("/{ontology_id}/graph/cypher") +@router.post("/{ontology_id}/graph/cypher", dependencies=[Depends(_require_read)]) def run_cypher(ontology_id: str, body: CypherRequest): """执行 Cypher 查询 (只读校验 + 强制 ontology_id 过滤)""" from app.services.v2.graph.cypher_builder import validate_readonly_cypher @@ -180,15 +196,16 @@ def run_cypher(ontology_id: str, body: CypherRequest): return {"results": results, "neo4j_available": True} -@router.get("/{ontology_id}/graph/neighbors/{node_id}") +@router.get("/{ontology_id}/graph/neighbors/{node_id}", dependencies=[Depends(_require_read)]) def get_neighbors(ontology_id: str, node_id: str, depth: int = 1): """查询节点邻居""" svc = get_neo4j() if not svc.available: return {"nodes": [], "edges": [], "neo4j_available": False} query = f""" - MATCH (n)-[r*1..{min(depth, 5)}]-(m) + MATCH path = (n)-[r*1..{min(depth, 5)}]-(m) WHERE elementId(n) = $node_id AND n.ontology_id = $ontology_id + AND all(node IN nodes(path) WHERE node.ontology_id = $ontology_id) RETURN n, r, m LIMIT 100 """ results = svc.run_cypher(query, {"node_id": node_id, "ontology_id": ontology_id}) @@ -203,7 +220,7 @@ class NLQueryRequest(BaseModel): schema: dict = {} -@router.post("/{ontology_id}/graph/ask") +@router.post("/{ontology_id}/graph/ask", dependencies=[Depends(_require_read)]) def nl_query(ontology_id: str, body: NLQueryRequest): """自然语言 → Cypher → 图数据""" from app.services.v2.graph.nl2cypher import NL2CypherService @@ -231,7 +248,7 @@ def nl_query(ontology_id: str, body: NLQueryRequest): # ── 高级图分析 ───────────────────────────────────────────────────────── -@router.get("/{ontology_id}/graph/path") +@router.get("/{ontology_id}/graph/path", dependencies=[Depends(_require_read)]) def graph_path(ontology_id: str, src: str, tgt: str): """两节点间最短路径""" from app.services.v2.graph.graph_analytics import GraphAnalyticsService @@ -239,7 +256,7 @@ def graph_path(ontology_id: str, src: str, tgt: str): return svc.shortest_path(ontology_id, src, tgt) -@router.get("/{ontology_id}/graph/degree/{node_id}") +@router.get("/{ontology_id}/graph/degree/{node_id}", dependencies=[Depends(_require_read)]) def node_degree(ontology_id: str, node_id: str): """查询节点度数(入度 + 出度)""" from app.services.v2.graph.graph_analytics import GraphAnalyticsService @@ -247,7 +264,7 @@ def node_degree(ontology_id: str, node_id: str): return svc.node_degree(ontology_id, node_id) -@router.get("/{ontology_id}/graph/top-nodes") +@router.get("/{ontology_id}/graph/top-nodes", dependencies=[Depends(_require_read)]) def top_nodes(ontology_id: str, limit: int = 10): """返回连接数最多的 Top-N 节点""" from app.services.v2.graph.graph_analytics import GraphAnalyticsService @@ -255,7 +272,7 @@ def top_nodes(ontology_id: str, limit: int = 10): return {"nodes": svc.top_connected_nodes(ontology_id, limit)} -@router.post("/{ontology_id}/graph/sync") +@router.post("/{ontology_id}/graph/sync", dependencies=[Depends(_require_edit)]) def sync_graph(ontology_id: str, current_user: User = Depends(require_editor)): """将 SQLite 实体/关系全量同步到 Neo4j""" from app.database import SessionLocal diff --git a/backend/app/routers/v2/logic_actions.py b/backend/app/routers/v2/logic_actions.py index fb22b32c..509d8cf5 100644 --- a/backend/app/routers/v2/logic_actions.py +++ b/backend/app/routers/v2/logic_actions.py @@ -12,6 +12,7 @@ from app.models.user import User from app.models.v2.logic import OntologyLogicRule, OntologyStateMachine from app.models.v2.action import OntologyActionType, OntologyActionRun +from app.services.ontology_access import GrantDenied, require_project_grant router = APIRouter(dependencies=[Depends(get_current_user)]) @@ -23,6 +24,20 @@ def get_db(): db.close() +def _require_read(ontology_id: str, db: Session = Depends(get_db), actor: User = Depends(get_current_user)): + try: + require_project_grant(db, actor, ontology_id, "read") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + + +def _require_edit(ontology_id: str, db: Session = Depends(get_db), actor: User = Depends(require_editor)): + try: + require_project_grant(db, actor, ontology_id, "edit") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + + # ── Logic Rules ───────────────────────────────────────────────── class LogicRuleCreate(BaseModel): @@ -59,7 +74,7 @@ class LogicTestRequest(BaseModel): parameters: dict = {} -@router.get("/{ontology_id}/logic") +@router.get("/{ontology_id}/logic", dependencies=[Depends(_require_read)]) def list_logic_rules(ontology_id: str, logic_type: str = "", db: Session = Depends(get_db)): q = db.query(OntologyLogicRule).filter(OntologyLogicRule.ontology_id == ontology_id) if logic_type: @@ -73,7 +88,7 @@ def list_logic_rules(ontology_id: str, logic_type: str = "", db: Session = Depen for r in rules] -@router.post("/{ontology_id}/logic", status_code=201) +@router.post("/{ontology_id}/logic", status_code=201, dependencies=[Depends(_require_edit)]) def create_logic_rule(ontology_id: str, body: LogicRuleCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): def _write(): rule = OntologyLogicRule( @@ -87,7 +102,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2logic.create", callback=_write) -@router.put("/{ontology_id}/logic/{rule_id}") +@router.put("/{ontology_id}/logic/{rule_id}", dependencies=[Depends(_require_edit)]) def update_logic_rule(ontology_id: str, rule_id: str, body: LogicRuleCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): rule = db.query(OntologyLogicRule).filter( OntologyLogicRule.id == rule_id, OntologyLogicRule.ontology_id == ontology_id @@ -102,7 +117,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2logic.update", callback=_write) -@router.patch("/{ontology_id}/logic/{rule_id}") +@router.patch("/{ontology_id}/logic/{rule_id}", dependencies=[Depends(_require_edit)]) def patch_logic_rule(ontology_id: str, rule_id: str, body: LogicRulePatch, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): rule = db.query(OntologyLogicRule).filter( OntologyLogicRule.id == rule_id, OntologyLogicRule.ontology_id == ontology_id @@ -121,7 +136,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2logic.patch", callback=_write) -@router.post("/{ontology_id}/logic/{rule_id}/review") +@router.post("/{ontology_id}/logic/{rule_id}/review", dependencies=[Depends(_require_edit)]) def review_logic_rule(ontology_id: str, rule_id: str, body: LogicReviewRequest, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): rule = db.query(OntologyLogicRule).filter( OntologyLogicRule.id == rule_id, OntologyLogicRule.ontology_id == ontology_id @@ -147,7 +162,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2logic.review", callback=_write) -@router.post("/{ontology_id}/logic/{rule_id}/test") +@router.post("/{ontology_id}/logic/{rule_id}/test", dependencies=[Depends(_require_read)]) def test_logic_rule(ontology_id: str, rule_id: str, body: LogicTestRequest, db: Session = Depends(get_db), _=Depends(require_editor)): rule = db.query(OntologyLogicRule).filter( OntologyLogicRule.id == rule_id, OntologyLogicRule.ontology_id == ontology_id @@ -157,7 +172,7 @@ def test_logic_rule(ontology_id: str, rule_id: str, body: LogicTestRequest, db: return {"rule_id": rule.id, **_evaluate_logic_rule(rule, body.row or {}, body.parameters or {})} -@router.delete("/{ontology_id}/logic/{rule_id}") +@router.delete("/{ontology_id}/logic/{rule_id}", dependencies=[Depends(_require_edit)]) def delete_logic_rule(ontology_id: str, rule_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): rule = db.query(OntologyLogicRule).filter( OntologyLogicRule.id == rule_id, OntologyLogicRule.ontology_id == ontology_id @@ -170,7 +185,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2logic.delete", callback=_write) -@router.post("/{ontology_id}/logic/publish") +@router.post("/{ontology_id}/logic/publish", dependencies=[Depends(_require_edit)]) def publish_logic_rules_v2(ontology_id: str, db: Session = Depends(get_db), _=Depends(require_editor)): """Direct tool publication is closed; use the Ontology compiler (P1C).""" raise HTTPException(status_code=403, detail="PUBLICATION_NOT_ENABLED") @@ -197,7 +212,7 @@ def publish_logic_rules_v2(ontology_id: str, db: Session = Depends(get_db), _=De # ── Logic: Discovery ──────────────────────────────────────────── -@router.post("/{ontology_id}/logic/discover") +@router.post("/{ontology_id}/logic/discover", dependencies=[Depends(_require_edit)]) def discover_logic_rules(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): """发现 Logic Rules(同步写入 v2 + v1 表,供前端 LogicTab 读取)""" from app.services.v2.mapping.mapping_service import MappingService @@ -268,7 +283,7 @@ def _result(): # ── State Machines ────────────────────────────────────────────── -@router.get("/{ontology_id}/state-machines") +@router.get("/{ontology_id}/state-machines", dependencies=[Depends(_require_read)]) def list_state_machines(ontology_id: str, db: Session = Depends(get_db)): machines = db.query(OntologyStateMachine).filter( OntologyStateMachine.ontology_id == ontology_id @@ -319,7 +334,7 @@ class ActionReviewRequest(BaseModel): notes: str = "" -@router.get("/{ontology_id}/actions") +@router.get("/{ontology_id}/actions", dependencies=[Depends(_require_read)]) def list_action_types(ontology_id: str, category: str = "", db: Session = Depends(get_db)): q = db.query(OntologyActionType).filter(OntologyActionType.ontology_id == ontology_id) if category: @@ -334,7 +349,7 @@ def list_action_types(ontology_id: str, category: str = "", db: Session = Depend "created_at": a.created_at.isoformat() if a.created_at else None} for a in actions] -@router.post("/{ontology_id}/actions", status_code=201) +@router.post("/{ontology_id}/actions", status_code=201, dependencies=[Depends(_require_edit)]) def create_action_type(ontology_id: str, body: ActionTypeCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): def _write(): act = OntologyActionType( @@ -349,7 +364,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2action.create", callback=_write) -@router.patch("/{ontology_id}/actions/{action_id}") +@router.patch("/{ontology_id}/actions/{action_id}", dependencies=[Depends(_require_edit)]) def patch_action_type(ontology_id: str, action_id: str, body: ActionTypePatch, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): act = db.query(OntologyActionType).filter( OntologyActionType.id == action_id, OntologyActionType.ontology_id == ontology_id @@ -368,7 +383,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2action.patch", callback=_write) -@router.post("/{ontology_id}/actions/{action_id}/review") +@router.post("/{ontology_id}/actions/{action_id}/review", dependencies=[Depends(_require_edit)]) def review_action_type(ontology_id: str, action_id: str, body: ActionReviewRequest, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): act = db.query(OntologyActionType).filter( OntologyActionType.id == action_id, OntologyActionType.ontology_id == ontology_id @@ -399,7 +414,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2action.review", callback=_write) -@router.post("/{ontology_id}/actions/discover") +@router.post("/{ontology_id}/actions/discover", dependencies=[Depends(_require_edit)]) def discover_actions(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): """发现 Actions(同步写入 v2 + v1 表,供前端 ActionsTab 读取)""" from app.services.v2.mapping.mapping_service import MappingService @@ -457,7 +472,7 @@ def _result(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2action.discover", callback=_result) -@router.delete("/{ontology_id}/actions/{action_id}") +@router.delete("/{ontology_id}/actions/{action_id}", dependencies=[Depends(_require_edit)]) def delete_action_type(ontology_id: str, action_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)): act = db.query(OntologyActionType).filter( OntologyActionType.id == action_id, OntologyActionType.ontology_id == ontology_id @@ -470,7 +485,7 @@ def _write(): return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2action.delete", callback=_write) -@router.post("/{ontology_id}/actions/publish") +@router.post("/{ontology_id}/actions/publish", dependencies=[Depends(_require_edit)]) def publish_actions_v2(ontology_id: str, db: Session = Depends(get_db), _=Depends(require_editor)): """Direct tool publication is closed; use the Ontology compiler (P1C).""" raise HTTPException(status_code=403, detail="PUBLICATION_NOT_ENABLED") @@ -497,7 +512,7 @@ def publish_actions_v2(ontology_id: str, db: Session = Depends(get_db), _=Depend # ── Action Runs ───────────────────────────────────────────────── -@router.get("/{ontology_id}/action-runs") +@router.get("/{ontology_id}/action-runs", dependencies=[Depends(_require_read)]) def list_action_runs(ontology_id: str, limit: int = 20, db: Session = Depends(get_db)): runs = db.query(OntologyActionRun).filter( OntologyActionRun.ontology_id == ontology_id @@ -507,7 +522,7 @@ def list_action_runs(ontology_id: str, limit: int = 20, db: Session = Depends(ge "started_at": r.started_at.isoformat() if r.started_at else None} for r in runs] -@router.post("/{ontology_id}/actions/{action_id}/run") +@router.post("/{ontology_id}/actions/{action_id}/run", dependencies=[Depends(_require_edit)]) def run_action_type( ontology_id: str, action_id: str, diff --git a/backend/app/routers/v2/pipelines.py b/backend/app/routers/v2/pipelines.py index 5441dcd2..12f37211 100644 --- a/backend/app/routers/v2/pipelines.py +++ b/backend/app/routers/v2/pipelines.py @@ -4,6 +4,7 @@ from pydantic import BaseModel from sqlalchemy.orm import Session from typing import Optional +import copy from datetime import datetime, timezone from app.database import SessionLocal from app.deps import get_current_user, require_admin, require_editor @@ -22,6 +23,80 @@ def get_db(): db.close() +_SQL_SOURCE_TYPES = {"mysql", "postgres", "postgresql"} +_UNSUPPORTED_INLINE_SOURCE_TYPES = {"mongo", "mongodb", "rest", "rest_api"} +_UNSUPPORTED_INLINE_CONNECTOR_CODE = "UNSUPPORTED_INLINE_CONNECTOR" + + +def _unsupported_inline_connector_nodes(definition: Optional[dict]) -> list[tuple[str, str]]: + if not isinstance(definition, dict): + return [] + nodes = definition.get("nodes", []) + if not isinstance(nodes, list): + return [] + unsupported = [] + for node in nodes: + if not isinstance(node, dict) or node.get("type") != "connector": + continue + config = node.get("config") + if not isinstance(config, dict): + continue + source_type = str(config.get("source_type", "")).strip().lower() + if source_type in _UNSUPPORTED_INLINE_SOURCE_TYPES: + unsupported.append((str(node.get("id", "")), source_type)) + return unsupported + + +def _reject_unsupported_inline_connectors(definition: Optional[dict]) -> None: + if _unsupported_inline_connector_nodes(definition): + raise HTTPException(status_code=422, detail=_UNSUPPORTED_INLINE_CONNECTOR_CODE) + + +def _sanitize_definition(definition: Optional[dict]) -> Optional[dict]: + """Keep only saved-connection references for SQL connector nodes. + + Credentials come from a saved Connection at run time. Legacy inline + values are not needed by the worker and must never be persisted again. + """ + if definition is None: + return None + sanitized = copy.deepcopy(definition) + nodes = sanitized.get("nodes", []) if isinstance(sanitized, dict) else [] + if not isinstance(nodes, list): + return sanitized + for node in nodes: + if not isinstance(node, dict) or node.get("type") != "connector": + continue + config = node.get("config") + if not isinstance(config, dict): + continue + source_type = str(config.get("source_type", "")).lower() + if source_type not in _SQL_SOURCE_TYPES: + continue + node["config"] = { + key: value for key, value in config.items() + if key in {"source_type", "connection_id", "resource", "table", "sync_mode"} + } + return sanitized + + +def _public_definition(definition: Optional[dict]) -> Optional[dict]: + """Hide credentials in legacy definitions without mutating stored rows.""" + sanitized = _sanitize_definition(definition) + if not isinstance(sanitized, dict): + return sanitized + for node in sanitized.get("nodes", []): + if not isinstance(node, dict) or node.get("type") != "connector": + continue + config = node.get("config") + if not isinstance(config, dict): + continue + source_type = str(config.get("source_type", "")).strip().lower() + if source_type in _UNSUPPORTED_INLINE_SOURCE_TYPES: + node["config"] = {"source_type": source_type} + return sanitized + + # ── Pydantic Models ──────────────────────────────────────────────── class PipelineCreate(BaseModel): @@ -76,6 +151,7 @@ class ValidateResult(BaseModel): @router.post("", response_model=PipelineResponse, status_code=201, dependencies=[Depends(require_editor)]) def create_pipeline(body: PipelineCreate, db: Session = Depends(get_db)): """创建新 Pipeline。支持旧 steps 格式和新 nodes/edges DSL。""" + _reject_unsupported_inline_connectors(body.definition) # 重名校验 existing = db.query(Pipeline).filter( Pipeline.name == body.name, @@ -100,7 +176,7 @@ def create_pipeline(body: PipelineCreate, db: Session = Depends(get_db)): source_dataset_id=body.source_dataset_id, route=inferred_route or "A", # SQLite 该列有 NOT NULL 约束 spec=body.spec or {}, - definition=body.definition, + definition=_sanitize_definition(body.definition), status="draft", branch="main", version=1, @@ -159,8 +235,11 @@ def update_pipeline(pipeline_id: str, body: PipelineUpdate, db: Session = Depend if not pl: raise HTTPException(404, "Pipeline not found") + _reject_unsupported_inline_connectors(body.definition) update_data = body.model_dump(exclude_unset=True) for k, v in update_data.items(): + if k == "definition": + v = _sanitize_definition(v) setattr(pl, k, v) pl.updated_at = datetime.now(timezone.utc) db.commit() @@ -201,6 +280,14 @@ def validate_pipeline(pipeline_id: str, db: Session = Depends(get_db), _=Depends nodes = definition.get("nodes", []) edges = definition.get("edges", []) + for node_id, source_type in _unsupported_inline_connector_nodes(definition): + errors.append({ + "node_id": node_id, + "severity": "error", + "code": _UNSUPPORTED_INLINE_CONNECTOR_CODE, + "message": "MongoDB/REST inline connector nodes are unsupported in Builder; use File Upload in Builder and manage external connections in Data Connections.", + }) + if not nodes: errors.append({"node_id": "", "severity": "error", "message": "Pipeline 至少需要一个节点。"}) @@ -320,6 +407,7 @@ def run_pipeline(pipeline_id: str, db: Session = Depends(get_db), _=Depends(requ pl = db.query(Pipeline).filter(Pipeline.id == pipeline_id).first() if not pl: raise HTTPException(404, "Pipeline not found") + _reject_unsupported_inline_connectors(pl.definition) pl.status = "running" run = PipelineRun(pipeline_id=pipeline_id, status="pending", started_at=datetime.now(timezone.utc)) @@ -453,7 +541,7 @@ def _format_pipeline(pl: Pipeline) -> dict: "source_dataset_id": pl.source_dataset_id, "route": pl.route, "spec": pl.spec or {}, - "definition": pl.definition, + "definition": _public_definition(pl.definition), "status": pl.status or "draft", "branch": pl.branch or "main", "version": pl.version or 1, diff --git a/backend/app/routers/v2/search.py b/backend/app/routers/v2/search.py index 673fc31d..6fcaf540 100644 --- a/backend/app/routers/v2/search.py +++ b/backend/app/routers/v2/search.py @@ -1,13 +1,15 @@ """v2 Search API — 关键词/语义统一搜索""" from __future__ import annotations import json -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel from sqlalchemy import cast, or_, String as SAString from sqlalchemy.orm import Session from app.database import SessionLocal from app.deps import get_current_user from app.models.entity import Entity +from app.models.user import User +from app.services.ontology_access import GrantDenied, require_project_grant router = APIRouter(dependencies=[Depends(get_current_user)]) @@ -20,6 +22,13 @@ def get_db(): db.close() +def _require_read(ontology_id: str, db: Session = Depends(get_db), actor: User = Depends(get_current_user)): + try: + require_project_grant(db, actor, ontology_id, "read") + except GrantDenied as exc: + raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc + + class SearchRequest(BaseModel): query: str mode: str = "keyword" # keyword | semantic @@ -54,7 +63,7 @@ def _sql_keyword_search(db: Session, ontology_id: str, q: str, n: int) -> list[d ] -@router.get("/{ontology_id}/search/keyword") +@router.get("/{ontology_id}/search/keyword", dependencies=[Depends(_require_read)]) def keyword_search( ontology_id: str, q: str = Query(..., description="搜索词"), @@ -71,7 +80,7 @@ def keyword_search( return {"results": results, "chroma_available": True, "query": q} -@router.get("/{ontology_id}/search/semantic") +@router.get("/{ontology_id}/search/semantic", dependencies=[Depends(_require_read)]) def semantic_search( ontology_id: str, q: str = Query(..., description="搜索词"), @@ -87,7 +96,7 @@ def semantic_search( return {"results": results, "chroma_available": True, "query": q} -@router.post("/{ontology_id}/search") +@router.post("/{ontology_id}/search", dependencies=[Depends(_require_read)]) def unified_search(ontology_id: str, body: SearchRequest, db: Session = Depends(get_db)): """统一搜索端点""" from app.services.v2.vector.chroma_service import ChromaService diff --git a/backend/app/schemas/ontology.py b/backend/app/schemas/ontology.py index 1c9906e8..e284d559 100644 --- a/backend/app/schemas/ontology.py +++ b/backend/app/schemas/ontology.py @@ -33,6 +33,7 @@ class OntologyOut(BaseModel): version: str status: str build_mode: Optional[str] = "simple_llm" + is_dirty: bool = False created_by: str created_at: datetime updated_at: datetime diff --git a/backend/app/services/connection/config.py b/backend/app/services/connection/config.py new file mode 100644 index 00000000..c865d421 --- /dev/null +++ b/backend/app/services/connection/config.py @@ -0,0 +1,96 @@ +"""Shared connection configuration handling. + +Connection records keep their credentials inside an encrypted JSON envelope. +Every runtime path that opens a source database goes through this module so a +saved connection and a refresh worker build the same SQLAlchemy URL. +""" +from __future__ import annotations + +import json +from typing import Any, Mapping +from urllib.parse import quote + + +SQL_KINDS = {"mysql", "postgres", "postgresql"} + + +class ConnectionConfigError(ValueError): + """A saved connection could not be converted into runtime configuration.""" + + +def decrypt_connection_config(raw_config: Mapping[str, Any] | None) -> dict[str, Any]: + """Return a runtime config from either plaintext or an encrypted envelope. + + Plaintext is accepted for compatibility with old fixtures and direct + connector callers. Persisted API writes always use the encrypted form. + """ + raw = dict(raw_config or {}) + encrypted = raw.get("_encrypted") + if not encrypted: + return raw + try: + from app.services import encryption_service + + decoded = json.loads(encryption_service.decrypt(str(encrypted))) + except Exception as exc: + raise ConnectionConfigError("connection configuration could not be decrypted") from exc + if not isinstance(decoded, dict): + raise ConnectionConfigError("connection configuration must be a JSON object") + return decoded + + +def _sql_url(raw_config: Mapping[str, Any], kind: str) -> str: + existing = raw_config.get("connection_string") + if existing: + return str(existing) + + host = str(raw_config.get("host") or "localhost") + port = raw_config.get("port") or (3306 if kind == "mysql" else 5432) + try: + port_value = int(port) + except (TypeError, ValueError) as exc: + raise ConnectionConfigError("database port must be a number") from exc + if not 1 <= port_value <= 65535: + raise ConnectionConfigError("database port must be between 1 and 65535") + + user = str(raw_config.get("user") or "") + password = str(raw_config.get("password") or "") + database = str(raw_config.get("database") or "") + if not user or not database: + raise ConnectionConfigError("database user and database name are required") + + scheme = "mysql+pymysql" if kind == "mysql" else "postgresql+psycopg2" + # Keep the host as supplied (including bracketed IPv6 literals), while + # encoding URL components that may contain @, :, /, #, or whitespace. + return ( + f"{scheme}://{quote(user, safe='')}:{quote(password, safe='')}" + f"@{host}:{port_value}/{quote(database, safe='')}" + ) + + +def build_connector_config( + kind: str, + raw_config: Mapping[str, Any] | None, + *, + resource: str | None = None, +) -> dict[str, Any]: + """Resolve persisted config and add the runtime DSN for SQL sources.""" + normalized_kind = "postgres" if kind == "postgresql" else kind + config = decrypt_connection_config(raw_config) + if normalized_kind in SQL_KINDS: + config["connection_string"] = _sql_url(config, normalized_kind) + if resource is not None: + config["resource"] = resource + return config + + +def safe_connection_error(exc: BaseException) -> str: + """Return useful connector detail while redacting common URL secrets.""" + message = str(exc) or exc.__class__.__name__ + # SQLAlchemy normally masks passwords, but drivers occasionally include a + # raw DSN in their exception. Avoid leaking password material in API + # responses and logs when that happens. + import re + + message = re.sub(r"(://[^:/@]+:)[^@]+(@)", r"\1***\2", message) + return message[:500] diff --git a/backend/app/services/connection/sql_connector.py b/backend/app/services/connection/sql_connector.py index c15b0abe..7b54171e 100644 --- a/backend/app/services/connection/sql_connector.py +++ b/backend/app/services/connection/sql_connector.py @@ -8,6 +8,7 @@ from sqlalchemy import create_engine, inspect, text from app.services.connection.base import ConnectorBase, DeltaPage, records_to_delta_page +from app.services.connection.config import safe_connection_error from app.schemas.refresh import SourceCursor # 合法 SQL 标识符:字母、数字、下划线、点(schema.table);禁止空格/分号/引号/注释等 @@ -36,6 +37,7 @@ class SQLConnector(ConnectorBase): def __init__(self, config: dict): self._config = config self._engine = None + self.last_error: str | None = None def _get_engine(self): if self._engine is None: @@ -50,8 +52,10 @@ def test_connection(self) -> bool: try: with self._get_engine().connect() as conn: conn.execute(text("SELECT 1")) + self.last_error = None return True - except Exception: + except Exception as exc: + self.last_error = safe_connection_error(exc) return False def list_resources(self) -> list[str]: diff --git a/backend/app/services/graph_diagnostics.py b/backend/app/services/graph_diagnostics.py new file mode 100644 index 00000000..639ca470 --- /dev/null +++ b/backend/app/services/graph_diagnostics.py @@ -0,0 +1,68 @@ +"""Graph connectivity diagnostics (graph-engineering playbook, Section V.E): +after entity resolution, a few structural signals reveal whether resolution +actually worked. A single connected component means every entity is +reachable from every other — fragmented islands mean surface-form variants +that should have merged did not. The edge/node ratio gives a density read; +the highest-degree entities are the hub nodes a future summarization stage +would prioritize. + +Pure functions only — no database access, no LLM calls — so this is callable +both from the extraction pipeline and, later, on demand for any ontology. +""" + + +def compute_graph_diagnostics(entities: list[dict], edges: list[tuple[str, str]], top_n: int = 5) -> dict: + """`entities`: [{"id": ..., "name_cn": ...}, ...] every entity in the + ontology. `edges`: [(source_id, target_id), ...] every relation, + direction ignored (weak connectivity, per the playbook).""" + ids = [e["id"] for e in entities if e.get("id")] + names = {e["id"]: e.get("name_cn") or e["id"] for e in entities if e.get("id")} + id_set = set(ids) + + adjacency: dict[str, list[str]] = {i: [] for i in ids} + degree: dict[str, int] = {i: 0 for i in ids} + edge_count = 0 + for source, target in edges: + if source not in id_set or target not in id_set: + continue # a dangling reference is a separate concern (P0 validation) + adjacency[source].append(target) + adjacency[target].append(source) + degree[source] += 1 + degree[target] += 1 + edge_count += 1 + + components = _weakly_connected_components(ids, adjacency) + component_sizes = sorted((len(c) for c in components), reverse=True) + isolated = [i for i in ids if degree[i] == 0] + hubs = sorted(ids, key=lambda i: degree[i], reverse=True)[:top_n] + + return { + "entity_count": len(ids), + "edge_count": edge_count, + "edge_node_ratio": round(edge_count / len(ids), 2) if ids else 0.0, + "connected_components": len(components), + "largest_component_size": component_sizes[0] if component_sizes else 0, + "isolated_entity_count": len(isolated), + "isolated_entities": [names[i] for i in isolated[:top_n]], + "hub_entities": [{"name_cn": names[i], "degree": degree[i]} for i in hubs if degree[i] > 0], + } + + +def _weakly_connected_components(ids: list[str], adjacency: dict[str, list[str]]) -> list[set]: + seen: set[str] = set() + components: list[set] = [] + for start in ids: + if start in seen: + continue + component = {start} + stack = [start] + seen.add(start) + while stack: + node = stack.pop() + for neighbor in adjacency.get(node, []): + if neighbor not in seen: + seen.add(neighbor) + component.add(neighbor) + stack.append(neighbor) + components.append(component) + return components diff --git a/backend/app/services/llm_service.py b/backend/app/services/llm_service.py index 7f9bfdf6..5575ad32 100644 --- a/backend/app/services/llm_service.py +++ b/backend/app/services/llm_service.py @@ -31,7 +31,12 @@ def extract_ontology(text: str, prompt_content: str, model_config: dict, model_n "每条关系的 source/target 必须取自已提取的实体\n" "5. 逻辑规则按 Palantir Ontology Functions 模型分类为 derived_property/aggregation/" "complex_edit/external_query 之一,写入 function_type + definition 字段,禁止输出 " - "IF-THEN 自然语言公式\n\n" + "IF-THEN 自然语言公式\n" + "6. 业务规则、政策、判断条件、定价/审批/校验类规则(如\"促销定价规则\")不是实体," + "禁止作为 entity 或 instance 输出,必须放入 logic_rules 数组\n" + "7. 属性的枚举值/分级/分档/阶段标签(如信用分层的\"A层\"\"B层\"、逾期阶段的\"M0\"\"M1\"、" + "融资轮次的\"A轮融资\")不是独立概念也不是命名实例,不要为它们创建 entity 或 instance," + "只需体现为所属概念某个属性的枚举值\n\n" f"文档内容:\n\n{text}" )}, ] @@ -61,7 +66,7 @@ def normalize_extracted_ontology(parsed: Any) -> dict: if not isinstance(entity, dict): continue name = _clean_entity_name(entity.get("name_cn")) - if not name or _is_junk_entity_name(name): + if not name or _is_junk_entity_name(name) or _looks_like_rule_name(name) or _looks_like_enum_value(name): continue key = _normalize_name(name) canonical = seen_names.get(key) @@ -140,14 +145,47 @@ def _is_junk_entity_name(name: str) -> bool: return False -def resolve_entities(entities: list[dict], model_config: dict, model_name: str) -> dict[str, str]: +_RULE_NAME_SUFFIXES = ("规则", "政策", "办法", "制度", "准则", "细则", "规程") + + +def _looks_like_rule_name(name: str) -> bool: + """A business rule/policy (e.g. "促销定价规则") is not an entity — it + belongs in logic_rules, not the concept graph.""" + return name.endswith(_RULE_NAME_SUFFIXES) + + +# A letter (+ optional 1-2 digits) followed by a small closed set of +# tier/stage/round markers, and nothing else — the shape of an enumeration +# VALUE of some concept's property (信用分层's "A层", 逾期阶段's "M0"/"M1+", +# 融资轮次's "D轮融资"), not a concept in its own right. +_ENUM_VALUE_PATTERN = re.compile( + r"^[A-Za-z]{1,2}[+-]?(轮融资|阶段|层|级|档|类)$|^M\d{1,2}\+?(阶段)?$" +) + + +def _looks_like_enum_value(name: str) -> bool: + return bool(_ENUM_VALUE_PATTERN.match(name)) + + +def resolve_entities( + entities: list[dict], model_config: dict, model_name: str, + existing_entities: list[dict] | None = None, +) -> dict[str, str]: """Cross-document entity resolution (graph-engineering playbook, resolution stage): within each entity type, cluster surface-form variants of the same real-world concept using the extraction-time descriptions as disambiguation context — catches cases exact-name matching misses (abbreviation vs full name, alias vs formal name across files). Returns an alias map {raw_name_cn: canonical_name_cn} covering every input name; unmatched names - and failed calls fall back to identity so no entity is ever silently lost.""" + and failed calls fall back to identity so no entity is ever silently lost. + + `existing_entities` (playbook incremental-update guidance: resolve new + entities against the existing canonical set, not against each other) are + entities already saved on this ontology from a prior run. They are never + renamed — if a new entity clusters with one of them, that existing + entity's name_cn is forced as the canonical, so re-running extraction + never silently relabels an already-published entity out from under its + existing relations/instances.""" alias_map: dict[str, str] = {} by_type: dict[str, list[dict]] = {} for e in entities: @@ -157,21 +195,43 @@ def resolve_entities(entities: list[dict], model_config: dict, model_name: str) alias_map[name] = name by_type.setdefault(e.get("type") or "", []).append(e) + existing_by_type: dict[str, list[dict]] = {} + existing_names: set[str] = set() + for e in existing_entities or []: + name = e.get("name_cn") + if not name: + continue + existing_names.add(name) + existing_by_type.setdefault(e.get("type") or "", []).append(e) + provider = model_config.get("provider", "openai") api_key = model_config.get("api_key", "") api_base = model_config.get("api_base") for etype, group in by_type.items(): - if len(group) < 2: + existing_group = existing_by_type.get(etype, []) + if len(group) + len(existing_group) < 2: continue entity_list = "\n".join( f"- {e['name_cn']}: {(e.get('description') or '').strip()[:100]}" for e in group ) + if existing_group: + entity_list += "\n" + "\n".join( + f"- [已存在] {e['name_cn']}: {(e.get('description') or '').strip()[:100]}" + for e in existing_group + ) + incremental_rule = ( + "\n标记为 [已存在] 的实体是本体里已经发布的规范实体:如果某个未标记的新实体与其属于" + "同一概念,该簇的 canonical 必须使用 [已存在] 实体的名字(去掉标记),绝不能改名或" + "另选一个新名字。" + if existing_group else "" + ) system_prompt = ( f"你是实体消歧专家。下面是同一类型({etype or '未分类'})从文档中提取的实体," "其中部分可能是同一概念的不同写法(如全称/简称、中英文并列、别名)。请聚类:" "每个输入名称必须出现在且仅出现在一个簇的 aliases 中;确实不同的概念各自单独成簇;" - "结合描述判断,不要仅凭字面相似合并不同概念;canonical 取信息最完整、无歧义的写法。\n\n" + "结合描述判断,不要仅凭字面相似合并不同概念;canonical 取信息最完整、无歧义的写法。" + f"{incremental_rule}\n\n" '只返回 JSON:{"clusters": [{"canonical": "规范名", "aliases": ["写法1", "写法2"]}]}' ) messages = [ @@ -183,15 +243,25 @@ def resolve_entities(entities: list[dict], model_config: dict, model_name: str) except Exception: continue # resolution is best-effort; keep identity mapping on failure group_names = {e["name_cn"] for e in group} + existing_group_names = {e["name_cn"] for e in existing_group} + all_names = group_names | existing_group_names for cluster in (parsed.get("clusters") or []) if isinstance(parsed, dict) else []: if not isinstance(cluster, dict): continue canonical = _clean_entity_name(cluster.get("canonical")) - if not canonical or canonical not in group_names: + aliases = [_clean_entity_name(a) for a in (cluster.get("aliases") or [])] + aliases = [a for a in aliases if a and a in all_names] + # an existing (already-published) entity in this cluster always + # wins the canonical slot, regardless of what the model picked + existing_in_cluster = [a for a in aliases if a in existing_group_names] + ( + [canonical] if canonical in existing_group_names else [] + ) + if existing_in_cluster: + canonical = existing_in_cluster[0] + elif not canonical or canonical not in group_names: continue - for alias in cluster.get("aliases") or []: - alias = _clean_entity_name(alias) - if alias and alias in group_names: + for alias in aliases: + if alias in group_names: # only remap names from this run's own entities alias_map[alias] = canonical return alias_map @@ -319,6 +389,51 @@ def infer_relations(entities: list, existing_relations: list, text: str, return [] # relation inference failure is non-fatal +def summarize_entity( + name: str, description: str, relations_text: str, model_config: dict, model_name: str, +) -> dict | None: + """Hub-node profile synthesis (graph-engineering playbook, Section V.A): + for a high-degree entity, synthesize a 2-3 paragraph summary, 3-5 atomic + traceable key facts, and a time range from its grounded description and + its known graph relations. Returns None on failure — summarization is + best-effort and never blocks extraction.""" + provider = model_config.get("provider", "openai") + api_key = model_config.get("api_key", "") + api_base = model_config.get("api_base") + + system_prompt = ( + f"为本体实体「{name}」生成知识图谱画像。\n\n" + f"已知描述:{description or '(无)'}\n\n" + f"该实体在图中的已知关系:\n{relations_text or '(无)'}\n\n" + "基于以上信息写一段2-3段的事实性综合描述;如信息有冲突,优先采用更具体的说法;" + "提炼3-5条可溯源到上述描述/关系的原子关键事实;不要编造描述和关系之外没有支持的事实;" + "时间范围用 YYYY 或 YYYY-MM 格式,没有明确时间信息就填 \"unknown\"。\n\n" + '只返回 JSON:{"summary": "...", "key_facts": ["...", "..."], ' + '"time_range": {"start": "...", "end": "..."}}' + ) + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": f"实体:{name}"}, + ] + try: + parsed = _parse_response(_call_llm(provider, api_key, api_base, model_name, messages)) + except Exception: + return None + if not isinstance(parsed, dict): + return None + time_range = parsed.get("time_range") + if not isinstance(time_range, dict): + time_range = {"start": "unknown", "end": "unknown"} + return { + "summary": str(parsed.get("summary") or ""), + "key_facts": [f for f in (parsed.get("key_facts") or []) if isinstance(f, str)][:5], + "time_range": { + "start": str(time_range.get("start") or "unknown"), + "end": str(time_range.get("end") or "unknown"), + }, + } + + def _call_llm(provider: str, api_key: str, api_base: str | None, model: str, messages: list, json_mode: bool = True) -> str: # Stable seed for reproducibility: derived from message content so same input → same seed. import hashlib as _hashlib, json as _json diff --git a/backend/app/services/ontology_access.py b/backend/app/services/ontology_access.py index 6fa7feea..a25b5fdb 100644 --- a/backend/app/services/ontology_access.py +++ b/backend/app/services/ontology_access.py @@ -21,6 +21,8 @@ from sqlalchemy.orm import Session from app.models.ontology_access import OntologyProjectAccessGrant +from app.models.ontology import OntologyProject +from app.models.user import User from app.services.auth_service import get_user_by_id from app.services.governance_audit import enqueue_audit @@ -78,7 +80,7 @@ def _enqueue_audit(db: Session, *, security_domain_id: str, correlation_id: str, def current_grant(db: Session, ontology_id: str, user_id: str) -> dict | None: row = db.execute( sa.text( - "SELECT id, ontology_id, user_id, capabilities, revision, status, created_by, " + "SELECT id, ontology_id, user_id, security_domain_id, capabilities, revision, status, created_by, " "valid_from, valid_until, revoked_by, created_at, updated_at, revoked_at " "FROM ontology_project_access_grants " "WHERE ontology_id = :o AND user_id = :u AND status = 'active'" @@ -226,10 +228,10 @@ def backfill_creator_grants(db: Session) -> dict: def _require_actor(db: Session, actor_id: str, ontology_id: str) -> dict: - grant = current_grant(db, ontology_id, actor_id) - if grant is None or "edit" not in grant["capabilities"]: - raise GrantDenied("actor lacks edit on this ontology") - return grant + actor = get_user_by_id(db, actor_id) + if actor is None: + raise GrantDenied("actor required") + return require_project_grant(db, actor, ontology_id, "edit") def _check_ceiling(role: str, capabilities: list[str]) -> None: @@ -426,14 +428,111 @@ def recover_owner(db: Session, *, ontology_id: str, base_finding_revision: int, def require_project_grant(db: Session, actor, ontology_id: str, capability: str) -> dict: - if actor is None: + if actor is None or not getattr(actor, "id", None) or not getattr(actor, "is_active", False): raise GrantDenied("actor required") - grant = current_grant(db, ontology_id, actor.id) - if grant is None or capability not in grant["capabilities"]: + + if capability not in CAPABILITIES: + raise GrantDenied("unknown capability") + + row = db.execute( + sa.text( + "SELECT g.id, g.ontology_id, g.user_id, g.security_domain_id, g.capabilities, " + "g.revision, g.status, g.created_by, g.valid_from, g.valid_until, g.revoked_by, " + "g.created_at, g.updated_at, g.revoked_at, " + "o.security_domain_id AS ontology_security_domain_id, " + "u.security_domain_id AS actor_security_domain_id, u.is_active AS actor_is_active, " + "u.role AS actor_role " + "FROM ontology_project_access_grants g " + "JOIN ontology_projects o ON o.id = g.ontology_id " + "JOIN users u ON u.id = g.user_id " + "WHERE g.ontology_id = :o AND g.user_id = :u" + ), + { + "o": ontology_id, + "u": actor.id, + }, + ).mappings().one_or_none() + if row is None: + raise GrantDenied("grant not held") + + grant = _row_dict(row) + now = datetime.now(timezone.utc) + valid_from = _as_utc(grant["valid_from"]) + valid_until = _as_utc(grant["valid_until"]) + actor_domain = getattr(actor, "security_domain_id", None) + if ( + not row["actor_is_active"] + or row["actor_security_domain_id"] != actor_domain + or row["ontology_security_domain_id"] != actor_domain + or grant["security_domain_id"] != actor_domain + or grant["status"] != "active" + or (valid_from is not None and valid_from > now) + or (valid_until is not None and now >= valid_until) + or capability not in grant["capabilities"] + or capability not in ROLE_CEILINGS.get(row["actor_role"], ROLE_CEILINGS["viewer"]) + ): raise GrantDenied("grant not held") return grant +def _as_utc(value): + """Treat naive database timestamps as UTC for grant validity checks.""" + if value is None: + return None + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def discoverable_ontology_ids(db: Session, actor): + """Return a SQL selectable of ontology IDs the actor may discover. + + This intentionally remains a selectable so callers can apply the grant + predicate before counting, paginating, or aggregating project rows. + Direct ``read`` grants without ``discover`` remain valid for ID-based + reads and therefore do not appear here. + """ + if actor is None or not getattr(actor, "id", None) or not getattr(actor, "is_active", False): + return sa.select(OntologyProject.id).where(sa.false()) + + capability = "discover" + if capability not in ROLE_CEILINGS.get(getattr(actor, "role", None), ROLE_CEILINGS["viewer"]): + return sa.select(OntologyProject.id).where(sa.false()) + + now = datetime.now(timezone.utc) + grant = OntologyProjectAccessGrant + user = User + ontology = OntologyProject + return ( + sa.select(ontology.id) + .join( + grant, + sa.and_( + grant.ontology_id == ontology.id, + grant.security_domain_id == ontology.security_domain_id, + ), + ) + .join( + user, + sa.and_( + user.id == grant.user_id, + user.security_domain_id == grant.security_domain_id, + ), + ) + .where( + grant.user_id == actor.id, + grant.security_domain_id == getattr(actor, "security_domain_id", None), + user.security_domain_id == getattr(actor, "security_domain_id", None), + user.is_active.is_(True), + grant.status == "active", + sa.or_(grant.valid_from.is_(None), grant.valid_from <= now), + sa.or_(grant.valid_until.is_(None), now < grant.valid_until), + sa.cast(grant.capabilities, sa.Text).like('%"discover"%'), + ) + .distinct() + ) + + def _json_list(values) -> str: import json diff --git a/backend/app/services/v2/dataset_service.py b/backend/app/services/v2/dataset_service.py index 14431bd6..d1cab5c8 100644 --- a/backend/app/services/v2/dataset_service.py +++ b/backend/app/services/v2/dataset_service.py @@ -110,11 +110,11 @@ def preview( self, dataset_id: str, version_no: int, - limit: int = 100, + limit: int | None = 100, *, version_id: str | None = None, ) -> list[dict]: - """CSV/JSON 数据预览。无需 DuckDB, 纯 Python 处理。""" + """CSV/JSON 数据预览。``limit=None`` 返回完整版本。""" query = self._db.query(DatasetVersion).filter(DatasetVersion.dataset_id == dataset_id) if version_id is not None: query = query.filter(DatasetVersion.id == version_id) @@ -156,7 +156,7 @@ def preview( if i < len(headers): row_dict[headers[i]] = val if val is not None else "" rows_list.append(row_dict) - if len(rows_list) >= limit: + if limit is not None and len(rows_list) >= limit: break wb.close() return rows_list @@ -168,7 +168,7 @@ def preview( reader = csv.DictReader(io.StringIO(text)) rows = [] for i, row in enumerate(reader): - if i >= limit: + if limit is not None and i >= limit: break rows.append(dict(row)) return rows diff --git a/backend/app/services/v2/graph/graph_analytics.py b/backend/app/services/v2/graph/graph_analytics.py index 76109c24..09634f97 100644 --- a/backend/app/services/v2/graph/graph_analytics.py +++ b/backend/app/services/v2/graph/graph_analytics.py @@ -20,6 +20,7 @@ def get_neighbors(self, ontology_id: str, node_id: str, depth: int = 1) -> dict: query = f""" MATCH path = (n)-[r*1..{depth}]-(m) WHERE elementId(n) = $node_id AND n.ontology_id = $ontology_id + AND all(node IN nodes(path) WHERE node.ontology_id = $ontology_id) RETURN n, relationships(path) AS rels, nodes(path) AS all_nodes LIMIT 100 """ @@ -56,6 +57,7 @@ def shortest_path(self, ontology_id: str, src_id: str, tgt_id: str) -> dict: WHERE elementId(s) = $src AND elementId(t) = $tgt AND s.ontology_id = $ontology_id AND t.ontology_id = $ontology_id MATCH p = shortestPath((s)-[*]-(t)) + WHERE all(node IN nodes(p) WHERE node.ontology_id = $ontology_id) RETURN [n IN nodes(p) | {id: elementId(n), labels: labels(n), name: n.name_cn}] AS path_nodes, length(p) AS path_length """ @@ -80,8 +82,10 @@ def node_degree(self, ontology_id: str, node_id: str) -> dict: query = """ MATCH (n) WHERE elementId(n) = $node_id AND n.ontology_id = $ontology_id - OPTIONAL MATCH (n)-[out]->() - OPTIONAL MATCH ()-[in]->(n) + OPTIONAL MATCH (n)-[out]->(out_neighbor) + WHERE out_neighbor.ontology_id = $ontology_id + OPTIONAL MATCH (in_neighbor)-[in]->(n) + WHERE in_neighbor.ontology_id = $ontology_id RETURN count(DISTINCT out) AS out_degree, count(DISTINCT in) AS in_degree """ try: @@ -102,8 +106,8 @@ def top_connected_nodes(self, ontology_id: str, limit: int = 10) -> list[dict]: if not self._neo4j.available: return [] query = """ - MATCH (n)-[r]-() - WHERE n.ontology_id = $ontology_id + MATCH (n)-[r]-(neighbor) + WHERE n.ontology_id = $ontology_id AND neighbor.ontology_id = $ontology_id RETURN elementId(n) AS node_id, labels(n) AS labels, n.name_cn AS name, diff --git a/backend/app/services/v2/incremental/operations.py b/backend/app/services/v2/incremental/operations.py index 69e6abbb..cc86601c 100644 --- a/backend/app/services/v2/incremental/operations.py +++ b/backend/app/services/v2/incremental/operations.py @@ -17,6 +17,7 @@ from sqlalchemy.orm import Session from app.models.v2.connection import Connection +from app.models.v2.dataset import Dataset from app.models.v2.pipeline import PipelineRunInput from app.models.v2.refresh import ( RefreshDeadLetter, @@ -39,12 +40,14 @@ collect_refresh_operability, ) from app.services.v2.scheduler.schedule_service import ( - DEFAULT_RESOURCE, ScheduleRequest, upsert_refresh_schedule, ) +_LEGACY_RESOURCE_SENTINEL = "__scheduled__" + + class RefreshRunView(BaseModel): """Non-secret, serializable view of a durable refresh run.""" @@ -147,25 +150,76 @@ def _source_state( ).scalars().first() -def _source_resource( - db: Session, *, source_id: str, resource: str | None = None, -) -> str: - if resource: - return resource - state = _source_state(db, source_id=source_id) - if state is not None: - return state.resource +def resolve_refresh_resource(db: Session, *, source_id: str) -> str: + """Resolve one concrete resource owned by a saved connection. + + The resolver deliberately evaluates one precedence level at a time. A + later source of truth cannot hide an ambiguity in an earlier one, and the + historical ``__scheduled__`` placeholder is never a candidate. + """ connection = db.get(Connection, source_id) if connection is None: raise RefreshError("SOURCE_NOT_FOUND", f"no source connection {source_id}") + + def candidates(values: list[object]) -> list[str]: + return sorted({str(value).strip() for value in values if value is not None and str(value).strip()}) + # Fail closed: an `_encrypted` config that cannot be decrypted must - # never be silently treated as "no resource hint configured" -- that - # would resolve to DEFAULT_RESOURCE and let a manual refresh trigger - # create a RefreshSourceState/RefreshRun for the wrong resource instead - # of surfacing the broken configuration. Credentials are never - # accepted from this request path; only the resource hint is read. + # never be silently treated as "no resource hint configured". Credentials + # are never accepted from this request path; only the resource hint is + # read. raw = _connection_configuration(connection) - return str(raw.get("resource") or raw.get("table") or DEFAULT_RESOURCE) + configured = candidates([raw.get("resource"), raw.get("table")]) + configured = [value for value in configured if value != _LEGACY_RESOURCE_SENTINEL] + if len(configured) > 1: + raise RefreshError( + "REFRESH_RESOURCE_AMBIGUOUS", + f"connection {source_id} has multiple configured resources", + ) + if configured: + return configured[0] + + state_resources = candidates(db.execute( + select(RefreshSourceState.resource).where( + RefreshSourceState.source_id == source_id, + RefreshSourceState.resource != _LEGACY_RESOURCE_SENTINEL, + ).distinct() + ).scalars().all()) + if len(state_resources) > 1: + raise RefreshError( + "REFRESH_RESOURCE_AMBIGUOUS", + f"connection {source_id} has multiple refresh resource states", + ) + if state_resources: + return state_resources[0] + + dataset_resources = candidates(db.execute( + select(Dataset.name).where( + Dataset.source_connection_id == source_id, + Dataset.name != _LEGACY_RESOURCE_SENTINEL, + ).distinct() + ).scalars().all()) + if len(dataset_resources) > 1: + raise RefreshError( + "REFRESH_RESOURCE_AMBIGUOUS", + f"connection {source_id} has multiple source datasets", + ) + if dataset_resources: + return dataset_resources[0] + + raise RefreshError( + "REFRESH_RESOURCE_REQUIRED", + f"connection {source_id} has no concrete refresh resource", + ) + + +def _source_resource( + db: Session, *, source_id: str, resource: str | None = None, +) -> str: + """Backwards-compatible explicit-resource adapter for API callers.""" + if resource: + return resource + return resolve_refresh_resource(db, source_id=source_id) def _connection_configuration(connection: Connection) -> dict[str, Any]: @@ -624,6 +678,7 @@ def get_refresh_health( __all__ = [ "RefreshRunView", "RefreshStatus", "RefreshScheduleView", "get_refresh_schedule", + "resolve_refresh_resource", "trigger_refresh", "cancel_refresh_run", "replay_refresh_run", "get_last_successful_refresh_run", "get_refresh_status", "get_refresh_health", "schedule_view", "update_refresh_schedule", diff --git a/backend/app/services/v2/incremental/polling.py b/backend/app/services/v2/incremental/polling.py index 1420936c..e060cfd2 100644 --- a/backend/app/services/v2/incremental/polling.py +++ b/backend/app/services/v2/incremental/polling.py @@ -43,6 +43,7 @@ ) from app.services.connection.base import DeltaPage, records_to_delta_page from app.services.connection.registry import get_connector +from app.services.connection.config import build_connector_config, decrypt_connection_config from app.services.v2.incremental.contract import ( _cursor_to_json, _lock_or_create_source_state, @@ -128,22 +129,16 @@ def _connector_for_source( def _source_configuration(state: RefreshSourceState | None, connection: Connection | None) -> dict: - raw = dict(connection.config or {}) if connection is not None else {} - encrypted = raw.get("_encrypted") - if encrypted: - try: - from app.services import encryption_service - - decrypted = encryption_service.decrypt(encrypted) - raw = json.loads(decrypted) - except Exception: - logger.warning("unable to decrypt refresh source configuration", exc_info=True) + raw = decrypt_connection_config(connection.config if connection is not None else {}) config = dict(raw) if state is not None and state.configuration: # Refresh revisions carry source-owned cursor/policy settings while # credentials and connection details remain on Connection. config.update(state.configuration) - return config + resource = state.resource if state is not None else config.get("resource") + if connection is None: + return config + return build_connector_config(connection.kind, config, resource=resource) def _as_utc(value: datetime | None) -> datetime | None: diff --git a/backend/app/services/v2/scheduler/cron_service.py b/backend/app/services/v2/scheduler/cron_service.py index 25e5443b..94f3fdb6 100644 --- a/backend/app/services/v2/scheduler/cron_service.py +++ b/backend/app/services/v2/scheduler/cron_service.py @@ -72,27 +72,10 @@ def schedule_connection_sync(self, connection_id: str, cron_expr: str, *, db: "S def schedule_pipeline_run(self, pipeline_id: str, cron_expr: str, *, db: "Session | None" = None, **schedule_kwargs) -> dict: - """为 Pipeline 注册定时运行任务。语义同 schedule_connection_sync。""" - cron_params = self.parse_cron(cron_expr) - if db is None: - logger.info(f"Pipeline {pipeline_id} cron 校验通过(未持久化): {cron_expr}") - return { - "pipeline_id": pipeline_id, - "cron": cron_expr, - "celery_crontab": cron_params, - "status": "validated", - } - - schedule = self._persist_schedule(db, target_type="pipeline", target_id=pipeline_id, - cron_expr=cron_expr, **schedule_kwargs) - logger.info(f"Pipeline {pipeline_id} 调度已持久化: {cron_expr}") - return { - "pipeline_id": pipeline_id, - "cron": cron_expr, - "celery_crontab": cron_params, - "status": "scheduled", - "next_due_at": schedule.next_due_at.isoformat() if schedule.next_due_at else None, - } + """Reject unsupported pipeline-target refresh schedules.""" + raise ValueError( + "PIPELINE_SCHEDULE_UNSUPPORTED: pipeline-target schedules are not supported" + ) @staticmethod def _persist_schedule(db: "Session", *, target_type: str, target_id: str, cron_expr: str, diff --git a/backend/app/services/v2/scheduler/schedule_service.py b/backend/app/services/v2/scheduler/schedule_service.py index f868e917..2e66b959 100644 --- a/backend/app/services/v2/scheduler/schedule_service.py +++ b/backend/app/services/v2/scheduler/schedule_service.py @@ -20,6 +20,7 @@ from __future__ import annotations import uuid +import logging from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import Callable, Mapping, Sequence @@ -38,6 +39,7 @@ ) _cron_service = CronService() +logger = logging.getLogger(__name__) # ScheduleRequest.target_type is the API-facing vocabulary ("connection" / # "pipeline"); RefreshSchedule.target_type persists the same vocabulary the @@ -47,11 +49,12 @@ _STORED_TARGET_TYPE = {"connection": "source", "pipeline": "pipeline"} _TASK_NAME_BY_STORED_TARGET_TYPE = {"source": "refresh.connection", "pipeline": "refresh.pipeline"} -# A connection/pipeline's schedule does not yet select a specific resource -# (multi-resource fan-out is a later task's concern); scheduled runs are -# scoped to this fixed placeholder resource until then. +# Retained as an import-compatible fixture value for older callers. New +# schedule dispatch must resolve a concrete resource before creating a run. DEFAULT_RESOURCE = "__scheduled__" +PIPELINE_SCHEDULE_UNSUPPORTED = "PIPELINE_SCHEDULE_UNSUPPORTED" + _MAX_RETRY_ATTEMPTS = 20 _DEFAULT_MAX_ATTEMPTS = 3 _MAX_RETRY_BACKOFF_SECONDS = 3600 @@ -131,6 +134,10 @@ def _validate_retry_policy(retry_policy: Mapping[str, object] | None) -> None: def _validate_request(request: ScheduleRequest) -> ZoneInfo: if request.target_type not in _STORED_TARGET_TYPE: raise ValueError(f"unsupported target_type: {request.target_type!r}") + if request.target_type == "pipeline": + raise ValueError( + f"{PIPELINE_SCHEDULE_UNSUPPORTED}: pipeline-target schedules are not supported" + ) if not request.target_id: raise ValueError("target_id is required") if not _cron_service.validate_cron(request.cron_expr): @@ -315,9 +322,34 @@ def _dispatch_one_schedule( db.commit() return None + if schedule.target_type == "pipeline": + # Rows written before pipeline-target scheduling was closed remain + # readable, but must never be turned into a polling run whose + # ``source_id`` is actually a pipeline id. + schedule.enabled = False + schedule.next_due_at = None + logger.warning( + "Skipping unsupported pipeline refresh schedule %s (%s): %s", + schedule.id, schedule.target_id, PIPELINE_SCHEDULE_UNSUPPORTED, + ) + db.commit() + return None + source_id = schedule.target_id - resource = DEFAULT_RESOURCE - task_name = _TASK_NAME_BY_STORED_TARGET_TYPE[schedule.target_type] + from app.services.v2.incremental.operations import resolve_refresh_resource + from app.schemas.refresh import RefreshError + + try: + resource = resolve_refresh_resource(db, source_id=source_id) + except RefreshError as exc: + logger.warning( + "Skipping refresh schedule %s for source %s: %s", + schedule.id, source_id, exc.reason_code, + ) + schedule.next_due_at = _next_fire_utc(schedule.cron_expression, tz, max(due_at, now)) + db.commit() + return None + task_name = _TASK_NAME_BY_STORED_TARGET_TYPE["source"] idempotency_key = f"schedule:{schedule.id}:{due_at.isoformat()}" run = _lock_schedule_occurrence( db, source_id=source_id, resource=resource, idempotency_key=idempotency_key, diff --git a/backend/app/tasks/extraction.py b/backend/app/tasks/extraction.py index fa4c21ae..35a8f52e 100644 --- a/backend/app/tasks/extraction.py +++ b/backend/app/tasks/extraction.py @@ -1,6 +1,7 @@ from app.tasks.celery_app import celery_app import logging +import re logger = logging.getLogger(__name__) @@ -8,6 +9,7 @@ from app.models import ( # noqa: E402, F401 user, ontology, file, prompt, model_config, entity, logic as logic_model, action, relation, extraction_task, rules_config, + entity_instance, entity_instance_relation, ) # ── 概念/实例分离指令 — 追加到所有提取 Prompt 之后 ─────────────────────────── @@ -20,7 +22,20 @@ 不要为文档中提到的每个具体命名对象(如具体公司名、具体人名、具体订单号、具体条款标题)单独创建 entity。 文档中提到的具体命名实例,放入新增的 instances 数组,通过 entity_type 字段关联到其所属概念的 name_cn: "instances": [{"entity_type": "所属概念的name_cn", "name_cn": "实例名称", "name_en": "可选", "properties": {"属性1": "值"}, "confidence": 0.9}] -relations 只在概念之间建立(如 供应商 -[supply]-> 产品),不要在具体实例之间建立关系。""" +relations 只在概念之间建立(如 供应商 -[supply]-> 产品),不要在具体实例之间建立关系。 + +如果两个具体命名实例之间存在文档明确写出的因果/治疗/相互作用等关系(如"2型糖尿病 导致 糖尿病肾病""阿司匹林 与 华法林 相互作用"), +放入新增的 instance_relations 数组,不要因为"relations 只在概念之间建立"而丢弃这类信息: +"instance_relations": [{"source": "实例名称", "target": "实例名称", "type": "causes|interacts_with|治疗|等英文或语义明确的关系类型", "confidence": 0.9}] +source/target 必须是 instances 数组里已出现的 name_cn,不要引用未提取的实例或概念实体本身。 + +属性的枚举值/分级/分档/阶段标签(如信用分层的"A层""B层""C层"、逾期阶段的"M0""M1""M2+"、 +融资轮次的"A轮融资""D轮融资")不是独立的概念,也不是需要单独建 instance 的命名对象——它们是所属 +概念(如"信用分层""逾期阶段""融资")某个属性的可选取值。不要为它们创建 entity,也不要创建 +instance;只需在所属概念的 properties 里以枚举值形式体现(如信用分层.properties.levels = ["A层","B层","C层"])。 + +业务规则、政策、判断条件、定价/审批/校验类规则(如"促销定价规则""审批超时规则""授信额度校验") +不是实体,禁止作为 entity 或 instance 输出,必须放入 logic_rules 数组。""" # ── Extraction results merge (方案A) ─────────────────────────────────────── @@ -31,6 +46,7 @@ def _merge_extraction_results(results: list[dict]) -> dict: all_logic: list[dict] = [] all_actions: list[dict] = [] all_instances: list[dict] = [] + all_instance_relations: list[dict] = [] # 收集全部 for r in results: @@ -53,6 +69,9 @@ def _merge_extraction_results(results: list[dict]) -> dict: for inst in (r.get("instances") or []): if isinstance(inst, dict): all_instances.append(inst) + for ir in (r.get("instance_relations") or []): + if isinstance(ir, dict): + all_instance_relations.append(ir) # 实体去重(按 name_cn,保留属性最丰富的) seen_entities: dict[str, dict] = {} @@ -116,12 +135,22 @@ def _merge_extraction_results(results: list[dict]) -> dict: seen_instances.add(key) instances.append(inst) + # 实例关系去重(source-target-type 唯一) + seen_inst_rels: set = set() + instance_relations = [] + for ir in all_instance_relations: + key = (ir.get("source", ""), ir.get("target", ""), ir.get("type", "")) + if key not in seen_inst_rels and key[0] and key[1]: + seen_inst_rels.add(key) + instance_relations.append(ir) + return { "entities": entities, "relations": relations, "logic_rules": list(seen_logic.values()), "actions": list(seen_actions.values()), "instances": instances, + "instance_relations": instance_relations, } @@ -241,6 +270,75 @@ def _fuzzy_resolve_entity(name: str, name_to_id: dict) -> str | None: return candidates[0][1] +# ── Tiered model selection (graph-engineering playbook, model selection) ──── +# Extraction is high-volume and schema-constrained — speed/cost dominate, so +# it always uses whatever model the user picked. Resolution and relation +# inference weigh conflicting evidence — reasoning quality dominates — so +# when the user picked an explicitly fast/cheap variant, we look for a +# stronger sibling already configured on the same model config and use that +# instead. No new user-facing setting: this reads only what's already in +# ModelConfig.models. +_FAST_MODEL_HINTS = ("flash", "mini", "haiku", "lite", "turbo", "nano", "fast") + + +def _select_reasoning_model(model_name: str, available_models: list) -> str: + """Pick the model to use for judgment-heavy stages (resolution, relation + inference). Falls back to `model_name` unchanged unless the user's pick + looks like a fast/cheap variant AND a non-fast sibling exists in the same + model config — so behavior is identical to today whenever no better + candidate is configured.""" + if not any(hint in model_name.lower() for hint in _FAST_MODEL_HINTS): + return model_name + for candidate in available_models or []: + if isinstance(candidate, str) and candidate != model_name and not any( + hint in candidate.lower() for hint in _FAST_MODEL_HINTS + ): + return candidate + return model_name + + +# ── Long-document chunking (graph-engineering playbook, scaling guidance) ─── +_CHUNK_MAX_CHARS = 12000 +_CHUNK_OVERLAP_CHARS = 500 +_SECTION_HEADER = re.compile(r"^#{1,3}\s+.*$", re.MULTILINE) + + +def _split_sections(md: str) -> list[str]: + """Split markdown into section chunks at heading boundaries. Falls back to + paragraph boundaries when the document has no headings at all.""" + starts = [m.start() for m in _SECTION_HEADER.finditer(md)] + if not starts or starts[0] != 0: + starts = [0] + starts + if len(starts) > 1: + return [md[s:e] for s, e in zip(starts, starts[1:] + [len(md)]) if md[s:e].strip()] + return [p for p in md.split("\n\n") if p.strip()] + + +def _chunk_document(md: str, max_chars: int = _CHUNK_MAX_CHARS, overlap_chars: int = _CHUNK_OVERLAP_CHARS) -> list[str]: + """Chunk a long document at section boundaries so a single extraction call + never has to hold the whole thing — entities/relations still land within + the same chunk as their context. Each chunk after the first repeats the + tail of the previous chunk as overlap, so a concept mentioned at a section + boundary isn't split from the relations described just after it. Short + documents (the common case) pass through unchanged as a single chunk.""" + if len(md) <= max_chars: + return [md] + + sections = _split_sections(md) + chunks: list[str] = [] + current = "" + for section in sections: + if current and len(current) + len(section) > max_chars: + chunks.append(current) + overlap = current[-overlap_chars:] if overlap_chars > 0 else "" + current = overlap + section + else: + current += section + if current.strip(): + chunks.append(current) + return chunks or [md] + + @celery_app.task(bind=True) def run_extraction(self, task_id: str): import app.models # noqa: F401 — register all tables for FK resolution @@ -286,6 +384,12 @@ def run_extraction(self, task_id: str): task.status = "failed"; task.error = "Model or prompt not found"; db.commit(); return model_name = task.parameters.get("model_name", "") + # resolution/relation-inference weigh conflicting evidence — use a + # stronger sibling model already configured on this model config when + # the user picked a fast/cheap variant for extraction (see + # _select_reasoning_model); falls back to model_name unchanged + # otherwise, so this is a no-op unless a better candidate exists + reasoning_model_name = _select_reasoning_model(model_name, model_cfg.models) config_dict = { "provider": model_cfg.provider, "api_key": decrypt(model_cfg.api_key_encrypted or ""), @@ -309,14 +413,19 @@ def _clean(text: str) -> str: max_workers = 1 # serial extraction to avoid OOM with large LLM payloads completed = 0 - task.progress = {"stage": f"extracting files 0/{len(valid_mds)} (parallel ×{max_workers})", "pct": 20} + # a long document is chunked at section boundaries first (see + # _chunk_document) — most documents are short and pass through as a + # single chunk, so this is a no-op for today's typical corpus + units = [(f, chunk) for f in valid_mds for chunk in _chunk_document(_clean(f.converted_md or ""))] + total_units = len(units) + + task.progress = {"stage": f"extracting files 0/{total_units} (parallel ×{max_workers})", "pct": 20} db.commit() with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = {} - for f in valid_mds: - md = _clean(f.converted_md or "") - futures[executor.submit(extract_ontology, md, prompt_content, config_dict, model_name)] = f + for f, chunk in units: + futures[executor.submit(extract_ontology, chunk, prompt_content, config_dict, model_name)] = f for future in as_completed(futures): f = futures[future] @@ -327,7 +436,7 @@ def _clean(text: str) -> str: except Exception: logger.exception("extract_ontology failed for file %s", getattr(f, "id", f)) completed += 1 - task.progress = {"stage": f"extracting files {completed}/{len(valid_mds)} (parallel ×{max_workers})", "pct": 20 + 35 * completed // len(valid_mds)} + task.progress = {"stage": f"extracting files {completed}/{total_units} (parallel ×{max_workers})", "pct": 20 + 35 * completed // total_units} db.commit() if all_results: @@ -337,11 +446,23 @@ def _clean(text: str) -> str: # so a concept extracted under different surface forms — whether # across files or within one document — would otherwise fracture # into separate nodes - if len(result.get("entities") or []) > 1: + new_entity_count = len(result.get("entities") or []) + # incremental-update guidance: resolve this run's new entities + # against the ontology's already-published entities too, so a + # re-run doesn't create a duplicate for a surface-form variant of + # something that already exists + existing_entities_for_resolution = [ + {"name_cn": e.name_cn, "description": e.description, "type": e.type} + for e in db.query(Entity).filter(Entity.ontology_id == task.ontology_id).all() + ] + if new_entity_count > 1 or (new_entity_count == 1 and existing_entities_for_resolution): task.progress = {"stage": "resolving entities", "pct": 58} db.commit() from app.services.llm_service import resolve_entities, apply_entity_resolution - alias_map = resolve_entities(result["entities"], config_dict, model_name) + alias_map = resolve_entities( + result["entities"], config_dict, reasoning_model_name, + existing_entities=existing_entities_for_resolution, + ) result = apply_entity_resolution(result, alias_map) else: # 全部失败时回退到合并文本单次提取 @@ -390,7 +511,7 @@ def _clean(text: str) -> str: db.commit() extra_rels = infer_relations( entities_extracted, relations_extracted, - combined_text, config_dict, model_name + combined_text, config_dict, reasoning_model_name ) if extra_rels: # Accept relations where both endpoints fuzzy-match a known entity name @@ -439,7 +560,10 @@ def _clean(text: str) -> str: ent = existing_ent_map[name_cn] # Always allow LLM to enrich description, properties, name_en, name_abbr if e_data.get("description"): ent.description = e_data["description"] - if props: ent.properties = props + # merge, don't replace — a prior run's hub summary + # (summary/key_facts/summary_degree) must survive a re-run + # that doesn't touch this entity's properties otherwise + if props: ent.properties = {**(ent.properties or {}), **props} if e_data.get("name_en"): ent.name_en = e_data["name_en"] if name_abbr: ent.name_abbr = name_abbr if e_data.get("type"): ent.type = e_data["type"] @@ -472,6 +596,10 @@ def _clean(text: str) -> str: # ── 写入实例数据(EntityInstance 表)— 挂在概念实体下,对齐 Pipeline Mapping ── from app.models.entity_instance import EntityInstance import hashlib as _hl + # instance_relations (below) needs to resolve a source/target instance + # name back to its stable id and owning concept entity id + instance_name_to_id: dict = {} + instance_id_to_entity_id: dict = {} for inst_data in result.get("instances", []): if not isinstance(inst_data, dict): continue @@ -506,11 +634,14 @@ def _clean(text: str) -> str: id=stable_id, entity_id=concept_id, ontology_id=task.ontology_id, row_identity=inst_name, row_data=inst_props, )) + instance_name_to_id[inst_name] = stable_id + instance_id_to_entity_id[stable_id] = concept_id db.commit() # ── Fix 2+4: upsert relations (by source_id, target_id, type) ──────── existing_rels = db.query(Relation).filter(Relation.ontology_id == task.ontology_id).all() existing_rel_set = {(r.source_entity, r.target_entity, r.type) for r in existing_rels} + relation_id_by_key: dict = {(r.source_entity, r.target_entity, r.type): r.id for r in existing_rels} for rel in result.get("relations", []): src_name = rel.get("source") or rel.get("source_entity", "") @@ -522,12 +653,70 @@ def _clean(text: str) -> str: if rel_type in ("关联", "未知", "相关", "其他", "") or not rel_type.isascii(): continue if src_id and tgt_id and (src_id, tgt_id, rel_type) not in existing_rel_set: + new_rel_id = str(uuid.uuid4()) db.add(Relation( - id=str(uuid.uuid4()), ontology_id=task.ontology_id, + id=new_rel_id, ontology_id=task.ontology_id, source_entity=src_id, target_entity=tgt_id, type=rel_type, confidence=rel.get("confidence", 0.85), )) existing_rel_set.add((src_id, tgt_id, rel_type)) + relation_id_by_key[(src_id, tgt_id, rel_type)] = new_rel_id + + # instance_relations below may reuse any of the relations just added + # via relation_id_by_key as their relation_definition_id FK — flush + # now so those rows exist before EntityInstanceRelation references them + db.flush() + + # ── instance-level relations (graph-engineering playbook): specific + # named things (e.g. 2型糖尿病, 阿司匹林) keep their own relation graph + # via EntityInstanceRelation, anchored to the concept-level Relation + # it instantiates (auto-created here if the LLM didn't also emit it) ── + from app.models.entity_instance_relation import EntityInstanceRelation + existing_inst_rels = db.query(EntityInstanceRelation).filter( + EntityInstanceRelation.ontology_id == task.ontology_id).all() + existing_inst_rel_set = { + (r.source_instance_id, r.target_instance_id, r.relation_definition_id) + for r in existing_inst_rels + } + + for ir in result.get("instance_relations", []): + if not isinstance(ir, dict): + continue + src_iid = _fuzzy_resolve_entity(ir.get("source", ""), instance_name_to_id) + tgt_iid = _fuzzy_resolve_entity(ir.get("target", ""), instance_name_to_id) + if not src_iid or not tgt_iid: + continue + ir_type = (ir.get("type") or "RELATED").strip() + if ir_type in ("关联", "未知", "相关", "其他", "") or not ir_type.isascii(): + continue + src_concept_id = instance_id_to_entity_id.get(src_iid) + tgt_concept_id = instance_id_to_entity_id.get(tgt_iid) + if not src_concept_id or not tgt_concept_id: + continue + concept_key = (src_concept_id, tgt_concept_id, ir_type) + rel_def_id = relation_id_by_key.get(concept_key) + if not rel_def_id: + rel_def_id = str(uuid.uuid4()) + db.add(Relation( + id=rel_def_id, ontology_id=task.ontology_id, + source_entity=src_concept_id, target_entity=tgt_concept_id, + type=ir_type, confidence=ir.get("confidence", 0.8), + )) + existing_rel_set.add(concept_key) + relation_id_by_key[concept_key] = rel_def_id + # the FK from entity_instance_relations to relations must see + # this row on insert — flush now rather than trust autoflush + # ordering across the two tables + db.flush() + edge_key = (src_iid, tgt_iid, rel_def_id) + if edge_key not in existing_inst_rel_set: + db.add(EntityInstanceRelation( + id=str(uuid.uuid4()), ontology_id=task.ontology_id, + source_instance_id=src_iid, target_instance_id=tgt_iid, + relation_definition_id=rel_def_id, + properties={"confidence": ir.get("confidence", 0.8)}, + )) + existing_inst_rel_set.add(edge_key) # ── Keyword matching helpers (unchanged) ───────────────────────────── all_entity_names = [ @@ -584,6 +773,12 @@ def _match_logic_rules(text: str, logic_name_to_id: dict) -> list: if not llm_linked: combined = " ".join(filter(None, [name_cn, r_data.get("definition",""), r_data.get("description","")])) llm_linked = _match_entities(combined, all_entity_names) + # A rule-category label (e.g. "反欺诈规则") is not a concept entity — + # it belongs in logic_rules, not entities — so a linked_entities + # reference to one is a dangling reference, same as a relation + # pointing at an entity the model never extracted; drop it rather + # than let it surface as a broken-reference validation warning. + llm_linked = [n for n in llm_linked if n in entity_name_to_id] if name_cn in existing_rule_map: rule = existing_rule_map[name_cn] @@ -623,6 +818,9 @@ def _match_logic_rules(text: str, logic_name_to_id: dict) -> list: if not linked_ents: combined = " ".join(filter(None, [name_cn, a_data.get("description","")])) linked_ents = _match_entities(combined, all_entity_names) + # Same dangling-reference cleanup as logic_rules above — a + # rule-category label is not a concept entity. + linked_ents = [n for n in linked_ents if n in entity_name_to_id] linked_logic_names = a_data.get("linked_logic_names", []) linked_ids = [logic_name_to_id[n] for n in linked_logic_names if n in logic_name_to_id] @@ -661,6 +859,50 @@ def _match_logic_rules(text: str, logic_name_to_id: dict) -> list: if project: project.status = "created" + # graph-engineering playbook diagnostics: computed over the ontology's + # full saved graph (not just this run's output), so a stale connected- + # components count from a prior run never masks today's actual state. + # session autoflush is off (app/database.py), so this run's own + # entity/relation adds above must be flushed before querying them back. + db.flush() + from app.services.graph_diagnostics import compute_graph_diagnostics + final_entities = db.query(Entity).filter(Entity.ontology_id == task.ontology_id).all() + final_relations = db.query(Relation).filter(Relation.ontology_id == task.ontology_id).all() + graph_diagnostics = compute_graph_diagnostics( + [{"id": e.id, "name_cn": e.name_cn} for e in final_entities], + [(r.source_entity, r.target_entity) for r in final_relations], + ) + task.validation_report = {**(task.validation_report or {}), "graph_diagnostics": graph_diagnostics} + + # hub-node summarization (playbook Section V.A): only entities with + # enough graph connectivity to benefit from cross-relation synthesis; + # re-summarizes only when degree changed since the last summary, so a + # re-run doesn't re-call the model for an unchanged hub every time + _HUB_DEGREE_THRESHOLD = 3 + degree_by_id: dict = {e.id: 0 for e in final_entities} + relation_lines_by_id: dict = {} + name_by_id = {e.id: e.name_cn for e in final_entities} + for r in final_relations: + if r.source_entity in degree_by_id: + degree_by_id[r.source_entity] += 1 + if r.target_entity in degree_by_id: + degree_by_id[r.target_entity] += 1 + line = f"({name_by_id.get(r.source_entity, r.source_entity)}) --[{r.type}]--> ({name_by_id.get(r.target_entity, r.target_entity)})" + relation_lines_by_id.setdefault(r.source_entity, []).append(line) + relation_lines_by_id.setdefault(r.target_entity, []).append(line) + + from app.services.llm_service import summarize_entity + for e in final_entities: + degree = degree_by_id.get(e.id, 0) + if degree < _HUB_DEGREE_THRESHOLD: + continue + if (e.properties or {}).get("summary_degree") == degree: + continue # graph neighborhood unchanged since last summary + relations_text = "\n".join(sorted(set(relation_lines_by_id.get(e.id, [])))) + profile = summarize_entity(e.name_cn, e.description or "", relations_text, config_dict, reasoning_model_name) + if profile: + e.properties = {**(e.properties or {}), **profile, "summary_degree": degree} + task.status = "completed" task.progress = {"stage": "done", "pct": 100} db.commit() @@ -676,6 +918,12 @@ def _match_logic_rules(text: str, logic_name_to_id: dict) -> list: if task: task.status = "failed" task.error = str(e) + # the router set the Ontology to "creating" when this task was + # queued — on failure it must go back to "draft" or it stays + # stuck showing "creating" forever with no task left to finish it + project = db.query(OntologyProject).filter(OntologyProject.id == task.ontology_id).first() + if project and project.status == "creating": + project.status = "draft" db.commit() except Exception: # session 可能已损坏,用新 session 兜底标记失败,避免任务永远卡在 running @@ -686,6 +934,9 @@ def _match_logic_rules(text: str, logic_name_to_id: dict) -> list: if task: task.status = "failed" task.error = str(e) + project = fresh_db.query(OntologyProject).filter(OntologyProject.id == task.ontology_id).first() + if project and project.status == "creating": + project.status = "draft" fresh_db.commit() fresh_db.close() except Exception: @@ -700,6 +951,8 @@ def _sync_neo4j(db, ontology_id: str) -> None: from app.services.v2.graph.neo4j_service import Neo4jService from app.models.entity import Entity from app.models.relation import Relation + from app.models.entity_instance import EntityInstance + from app.models.entity_instance_relation import EntityInstanceRelation svc = Neo4jService() if not svc.available: @@ -747,6 +1000,39 @@ def _sync_neo4j(db, ontology_id: str) -> None: props={"id": r.id, "ontology_id": ontology_id, "confidence": r.confidence or 0.85}, ) + # 具体命名实例(如 2型糖尿病、阿司匹林)作为独立节点同步,标签在概念类型 + # 后加 Instance 后缀,与概念节点区分开,避免图查询把概念和实例混为一谈 + instances = db.query(EntityInstance).filter(EntityInstance.ontology_id == ontology_id).all() + instance_label_map: dict[str, str] = {} + for inst in instances: + concept_type = entity_type_map.get(inst.entity_id, "OntologyEntity") + label = f"{concept_type}Instance" + instance_label_map[inst.id] = label + props = { + **(inst.row_data or {}), + "id": inst.id, + "ontology_id": ontology_id, + "entity_id": inst.entity_id, + "name_cn": inst.row_identity, + } + svc.upsert_entity(label, props, key_field="id") + + # 实例级关系(语义概念层面的 relation 已在上面写入;具体实例之间的边只 + # 存在于 Neo4j,不进 Postgres relations 表) + inst_relations = db.query(EntityInstanceRelation).filter( + EntityInstanceRelation.ontology_id == ontology_id).all() + relation_type_by_id = {r.id: r.type for r in relations} + for ir in inst_relations: + src_label = instance_label_map.get(ir.source_instance_id, "OntologyEntityInstance") + tgt_label = instance_label_map.get(ir.target_instance_id, "OntologyEntityInstance") + rel_type = (relation_type_by_id.get(ir.relation_definition_id) or "RELATED").upper().replace(" ", "_").replace("-", "_") + svc.upsert_relation( + src_label=src_label, src_key=ir.source_instance_id, + tgt_label=tgt_label, tgt_key=ir.target_instance_id, + rel_type=rel_type, + props={"id": ir.id, "ontology_id": ontology_id, **(ir.properties or {})}, + ) + svc.close() except Exception: logger.warning("Neo4j sync failed for ontology %s; extraction result unaffected", ontology_id, exc_info=True) diff --git a/backend/app/tasks/topology.py b/backend/app/tasks/topology.py index e30f1fa3..9f6bcdef 100644 --- a/backend/app/tasks/topology.py +++ b/backend/app/tasks/topology.py @@ -76,6 +76,19 @@ "agent.retention_purge": {"queue": QUEUE_HOUSEKEEPING}, } +# These four share one queue/worker pool. Without acks_late + a time limit, a +# single hung LLM call (or a worker dying mid-task) permanently wedges one of +# `--concurrency` slots with nothing to free it — every later task on this +# queue then sits "queued" forever with no worker left to pick it up. +ARTIFACT_EXTRACTION_TASK_NAMES = ( + "app.tasks.extraction.run_extraction", + "app.tasks.audit.run_audit", + "app.tasks.v2.mapping_apply.mapping_apply_task", + "app.tasks.v2.pipeline_run.pipeline_run_task", +) +ARTIFACT_EXTRACTION_SOFT_TIME_LIMIT_SECONDS = 1800 +ARTIFACT_EXTRACTION_HARD_TIME_LIMIT_SECONDS = 1860 + # ── Bounded refresh worker settings ───────────────────────────────────── @dataclass(frozen=True) @@ -227,3 +240,12 @@ def configure_celery_topology(celery_app: Celery) -> None: } for name in REFRESH_TASK_NAMES } + celery_app.conf.task_annotations.update({ + name: { + "acks_late": True, + "reject_on_worker_lost": True, + "soft_time_limit": ARTIFACT_EXTRACTION_SOFT_TIME_LIMIT_SECONDS, + "time_limit": ARTIFACT_EXTRACTION_HARD_TIME_LIMIT_SECONDS, + } + for name in ARTIFACT_EXTRACTION_TASK_NAMES + }) diff --git a/backend/app/tasks/v2/pipeline_run.py b/backend/app/tasks/v2/pipeline_run.py index 7326181d..5574d1b6 100644 --- a/backend/app/tasks/v2/pipeline_run.py +++ b/backend/app/tasks/v2/pipeline_run.py @@ -3,11 +3,14 @@ from datetime import datetime, timezone import logging from pathlib import Path -from typing import Sequence +from typing import Mapping, Sequence import uuid logger = logging.getLogger(__name__) +from app.services.connection.config import build_connector_config, safe_connection_error +from app.services.connection.registry import get_connector + def get_celery_app(): try: @@ -202,6 +205,7 @@ def _find_dataset_for_file(db, filename: str): def _collect_sources(db, pl) -> list[dict]: + from app.models.v2.connection import Connection from app.models.v2.dataset import Dataset sources: list[dict] = [] @@ -209,7 +213,29 @@ def _collect_sources(db, pl) -> list[dict]: for node in definition.get("nodes") or []: if node.get("type") != "connector": continue - for file_info in (node.get("config") or {}).get("files", []) or []: + config = node.get("config") or {} + source_type = str(config.get("source_type") or "").lower() + if source_type in {"mysql", "postgres", "postgresql"}: + connection_id = str(config.get("connection_id") or "") + resource = str(config.get("resource") or config.get("table") or "") + if not connection_id: + raise ValueError("Database connector must select a saved connection") + if not resource: + raise ValueError("Database connector must select a table") + connection = db.get(Connection, connection_id) + if connection is None: + raise ValueError(f"Saved connection not found: {connection_id}") + sources.append({ + "connection_id": connection.id, + "resource": resource, + "filename": resource, + "route": "A", + "kind": "structured", + "dataset_id": None, + }) + continue + + for file_info in config.get("files", []) or []: filename = file_info.get("name") or file_info.get("filename") or "" dataset_id = file_info.get("dataset_id") ds = db.query(Dataset).filter(Dataset.id == dataset_id).first() if dataset_id else None @@ -237,13 +263,146 @@ def _collect_sources(db, pl) -> list[dict]: seen: set[str] = set() unique_sources = [] for source in sources: - if source["dataset_id"] in seen: + key = source.get("dataset_id") or ( + f"connection:{source.get('connection_id')}:{source.get('resource')}" + ) + if key in seen: continue - seen.add(source["dataset_id"]) + seen.add(key) unique_sources.append(source) return unique_sources +def _existing_database_dataset(db, connection_id: str, resource: str): + from app.models.v2.dataset import Dataset + + return db.query(Dataset).filter( + Dataset.source_connection_id == connection_id, + Dataset.name == resource, + ).order_by(Dataset.created_at.desc()).first() + + +def _pinned_database_version(db, dataset_id: str, input_dataset_version_ids) -> object | None: + if not input_dataset_version_ids: + return None + from app.models.v2.dataset import DatasetVersion + + for version_id in input_dataset_version_ids: + version = db.get(DatasetVersion, version_id) + if version is not None and version.dataset_id == dataset_id: + return version + return None + + +def _resolve_pinned_input_versions(db, sources: list[dict], input_dataset_version_ids) -> dict[int, object]: + """Resolve one immutable input version for every source before any pull. + + A refresh-triggered pipeline must not silently fill a missing source pin + from a current/latest dataset version. SQL sources may not have a + materialized Dataset until their first unpinned run, so their candidate + Dataset rows are resolved by the saved connection/resource identity. + """ + if not input_dataset_version_ids: + return {} + + from app.models.v2.dataset import Dataset, DatasetVersion + + requested_ids = list(input_dataset_version_ids) + versions = db.query(DatasetVersion).filter(DatasetVersion.id.in_(set(requested_ids))).all() + versions_by_id = {version.id: version for version in versions} + resolved: dict[int, object] = {} + + for source in sources: + dataset_ids: set[str] = set() + if source.get("dataset_id"): + dataset_ids.add(source["dataset_id"]) + elif source.get("connection_id"): + datasets = db.query(Dataset).filter( + Dataset.source_connection_id == source["connection_id"], + Dataset.name == source["resource"], + ).all() + dataset_ids.update(dataset.id for dataset in datasets) + + matches = [ + versions_by_id[version_id] + for version_id in requested_ids + if version_id in versions_by_id + and versions_by_id[version_id].dataset_id in dataset_ids + ] + if len(matches) != 1: + source_name = source.get("resource") or source.get("filename") or "unknown" + raise ValueError( + "PINNED_INPUTS_INCOMPLETE: source " + f"{source_name!r} requires exactly one pinned dataset version" + ) + resolved[id(source)] = matches[0] + + return resolved + + +def _snapshot_database_source( + db, svc, source: dict, input_dataset_version_ids, selected_version=None, +) -> object: + """Materialize a selected SQL table as an immutable DatasetVersion. + + A refresh-triggered run can provide an exact pinned version; in that case + the live table is never read again. Manual runs without a pin read the + selected resource once and persist a new snapshot before Route A loads it. + """ + from app.models.v2.connection import Connection + from app.models.v2.dataset import Dataset + + connection = db.get(Connection, source["connection_id"]) + if connection is None: + raise ValueError(f"Saved connection not found: {source['connection_id']}") + dataset = _existing_database_dataset(db, connection.id, source["resource"]) + if selected_version is not None: + # A pinned SQL version is already immutable. Reuse it directly and + # recover the Dataset identity without touching the live connector. + if dataset is None: + dataset = db.get(Dataset, selected_version.dataset_id) + if ( + dataset is None + or dataset.id != selected_version.dataset_id + or dataset.source_connection_id != connection.id + or dataset.name != source["resource"] + ): + raise ValueError( + "PINNED_INPUTS_INCOMPLETE: pinned version does not belong to " + f"{connection.id}/{source['resource']}" + ) + source["dataset_id"] = dataset.id + return selected_version + if dataset is not None: + pinned = _pinned_database_version(db, dataset.id, input_dataset_version_ids) + if pinned is not None: + source["dataset_id"] = dataset.id + return pinned + + config = build_connector_config( + connection.kind, connection.config, resource=source["resource"], + ) + # A pipeline table selector is authoritative. Do not allow an optional + # saved query to redirect it to an arbitrary statement. + config.pop("query", None) + connector = get_connector(connection.kind, config) + rows = connector.pull_full(source["resource"]) + if isinstance(rows, Mapping): + rows = [dict(rows)] + else: + rows = [dict(row) for row in (rows or [])] + + if dataset is None: + dataset = svc.create_dataset( + name=source["resource"], kind="structured", connection_id=connection.id, + ) + version = svc.create_version( + dataset.id, _safe_csv_bytes(rows), rowcount=len(rows), + ) + source["dataset_id"] = dataset.id + return version + + def _load_source_rows( db, svc, @@ -273,8 +432,14 @@ def _load_source_rows( }] if version is None: return [] + # A database snapshot is already a bounded, immutable input selected by + # the pipeline. Loading only the preview cap would silently drop rows + # from the pipeline, so request the complete version for DB sources while + # retaining the existing preview cap for file sources. + preview_limit = None if source.get("connection_id") else limit return svc.preview( - source["dataset_id"], version.version_no, limit=limit, version_id=version.id, + source["dataset_id"], version.version_no, + limit=preview_limit, version_id=version.id, ) @@ -375,6 +540,21 @@ def _save_curated_outputs(db, svc, pl, source: dict, data: list[dict], ctx, mult return [_save_curated_dataset(db, svc, pl, source, data, ctx, multi_source)] +def _set_pipeline_status_if_latest(db, pipeline, run, status: str) -> bool: + """Update the pipeline summary only for the newest deterministic run.""" + from app.models.v2.pipeline import PipelineRun + + latest = db.query(PipelineRun).filter( + PipelineRun.pipeline_id == run.pipeline_id, + ).order_by( + PipelineRun.created_at.desc(), PipelineRun.id.desc(), + ).first() + if latest is None or latest.id != run.id: + return False + pipeline.status = status + return True + + def pipeline_run_task( pipeline_id: str, run_id: str, input_dataset_version_ids: Sequence[str] | None = None, @@ -382,20 +562,18 @@ def pipeline_run_task( """Pipeline 执行任务 — 支持 DAG 编译 + 节点状态追踪""" from app.database import SessionLocal from app.models.v2.pipeline import Pipeline, PipelineRun, PipelineRunInput - from app.models.v2.dataset import DatasetVersion from app.services.v2.pipeline.base import PipelineContext from app.services.v2.pipeline.dag_compiler import compile_definition from app.services.v2.dataset_service import DatasetService from app.services.v2.incremental.polling import select_pinned_dataset_version db = SessionLocal() + run = None + pl = None try: run = db.query(PipelineRun).filter(PipelineRun.id == run_id).first() if not run: return - run.status = "running" - run.started_at = datetime.now(timezone.utc) - db.commit() pl = db.query(Pipeline).filter(Pipeline.id == pipeline_id).first() if not pl: @@ -404,6 +582,11 @@ def pipeline_run_task( db.commit() return + run.status = "running" + run.started_at = datetime.now(timezone.utc) + _set_pipeline_status_if_latest(db, pl, run, "running") + db.commit() + # ── DAG 编译 ────────────────────────────────────────────── definition = pl.definition plan = compile_definition(definition) @@ -433,30 +616,40 @@ def set_node_status(nid: str, status: str): else: input_dataset_version_ids = list(input_dataset_version_ids) lineage_version_ids = list(input_dataset_version_ids) + pinned_versions = _resolve_pinned_input_versions( + db, sources, input_dataset_version_ids, + ) if input_dataset_version_ids: run.dataset_version_id = input_dataset_version_ids[0] db.commit() transform_route, runtime_spec = _pipeline_runtime_config(pl) - if sources and not pl.source_dataset_id: - pl.source_dataset_id = sources[0]["dataset_id"] - db.commit() - outputs = [] multi_source = len(sources) > 1 for source in sources: + pinned_version = pinned_versions.get(id(source)) + if source.get("connection_id"): + source_version = _snapshot_database_source( + db, svc, source, input_dataset_version_ids, + selected_version=pinned_version, + ) + else: + source_version = pinned_version + # SQL source identity is only known after its first snapshot. + # A multi-source pipeline cannot be represented by this one-source + # refresh binding; never silently bind it to the first input. + if not multi_source and not pl.source_dataset_id and source.get("dataset_id"): + pl.source_dataset_id = source["dataset_id"] + db.commit() source["route"] = _source_runtime_route(source, transform_route, pl.route) - source_version_ids = [] - for version_id in input_dataset_version_ids: - version = db.get(DatasetVersion, version_id) - if version is not None and version.dataset_id == source["dataset_id"]: - source_version_ids.append(version.id) - source_version = select_pinned_dataset_version( - db, - dataset_id=source["dataset_id"], - input_dataset_version_ids=source_version_ids or None, - ) + source_version_ids = [source_version.id] if pinned_version is not None else [] + if source_version is None: + source_version = select_pinned_dataset_version( + db, + dataset_id=source["dataset_id"], + input_dataset_version_ids=source_version_ids or None, + ) data = _load_source_rows( db, svc, source, input_dataset_version_ids=source_version_ids or None, @@ -544,17 +737,20 @@ def set_node_status(nid: str, status: str): "curated_dataset_id": curated_ids[0] if curated_ids else None, "curated_dataset_ids": curated_ids, } + _set_pipeline_status_if_latest(db, pl, run, "published") db.commit() except Exception as e: - logger.error(f"Pipeline run failed: {e}") + detail = safe_connection_error(e) + logger.error("Pipeline run failed: %s", detail) if run: run.status = "failed" - run.error_log = str(e) + run.error_log = detail run.finished_at = datetime.now(timezone.utc) stats = dict(run.stats or {}) stats.setdefault("node_status", {}) run.stats = stats + _set_pipeline_status_if_latest(db, pl, run, "failed") db.commit() finally: db.close() diff --git a/backend/app/tasks/v2/refresh_tasks.py b/backend/app/tasks/v2/refresh_tasks.py index b5b2138a..6bd262e6 100644 --- a/backend/app/tasks/v2/refresh_tasks.py +++ b/backend/app/tasks/v2/refresh_tasks.py @@ -27,12 +27,6 @@ logger = logging.getLogger(__name__) -# Placeholder resource scope for a connection/pipeline-level manual or -# scheduled refresh until multi-resource fan-out lands (matches -# schedule_service.DEFAULT_RESOURCE). -DEFAULT_RESOURCE = "__scheduled__" - - def _celery_send_task(task_name: str, args, queue: str) -> str: """Adapter matching `enqueue_refresh_run`'s `send_task` contract — Celery's own `send_task(name, args, kwargs, ...)` would otherwise bind a @@ -325,27 +319,28 @@ def create_manual_connection_run(db, connection_id: str): """ from app.models.v2.connection import Connection from app.schemas.refresh import RefreshError, RefreshPolicy - from app.services.v2.incremental.operations import _policy_for_source, _source_state + from app.services.v2.incremental.operations import ( + _policy_for_source, + _source_state, + resolve_refresh_resource, + ) from app.services.v2.incremental.contract import claim_refresh_run connection = db.get(Connection, connection_id) if connection is None: - # Preserve the historical internal helper behavior for callers that - # already provision source state without a Connection row. The - # authenticated route verifies the connection first; persisted - # policies are enforced whenever that source record exists. - policy = RefreshPolicy.BATCH - else: - configured_state = _source_state(db, source_id=connection_id) - policy = _policy_for_source(connection, configured_state) - if policy is RefreshPolicy.EVENT_DRIVEN: - raise RefreshError( - "EVENT_TRIGGER_REQUIRES_SIGNATURE", - "event-driven refreshes require the signed webhook route", - ) + raise RefreshError("SOURCE_NOT_FOUND", f"no source connection {connection_id}") + + resource = resolve_refresh_resource(db, source_id=connection_id) + configured_state = _source_state(db, source_id=connection_id, resource=resource) + policy = _policy_for_source(connection, configured_state) + if policy is RefreshPolicy.EVENT_DRIVEN: + raise RefreshError( + "EVENT_TRIGGER_REQUIRES_SIGNATURE", + "event-driven refreshes require the signed webhook route", + ) run = claim_refresh_run( - db, source_id=connection_id, resource=DEFAULT_RESOURCE, policy=policy, + db, source_id=connection_id, resource=resource, policy=policy, idempotency_key=f"manual:{uuid.uuid4()}", lease_owner="manual-sync", now=datetime.now(timezone.utc), ) diff --git "a/backend/evals/document_extraction/ground_truth/\344\276\233\345\272\224\351\223\276.entities.json" "b/backend/evals/document_extraction/ground_truth/\344\276\233\345\272\224\351\223\276.entities.json" new file mode 100644 index 00000000..c992b477 --- /dev/null +++ "b/backend/evals/document_extraction/ground_truth/\344\276\233\345\272\224\351\223\276.entities.json" @@ -0,0 +1,17 @@ +[ + "供应商", + "供应商分级", + "库存", + "物料", + "安全库存", + "采购申请", + "采购审批", + "质量控制", + "来料检验", + "供应商审计", + "物流", + "配送", + "承运商", + "质检部", + "董事会" +] diff --git "a/backend/evals/document_extraction/ground_truth/\344\277\241\350\264\267.entities.json" "b/backend/evals/document_extraction/ground_truth/\344\277\241\350\264\267.entities.json" new file mode 100644 index 00000000..d0709fdb --- /dev/null +++ "b/backend/evals/document_extraction/ground_truth/\344\277\241\350\264\267.entities.json" @@ -0,0 +1,24 @@ +[ + "借款人", + "金融机构资金方", + "信贷科技平台", + "信贷驱动服务", + "平台服务", + "重资本模式", + "轻资本模式", + "信用分层", + "信用评分", + "授信额度", + "授信审批", + "逾期阶段", + "贷后催收", + "风险准备金", + "获客渠道", + "转化漏斗", + "风险定价模型", + "客户生命周期", + "合规部", + "风控委员会", + "反欺诈黑名单", + "多头借贷指标" +] diff --git "a/backend/evals/document_extraction/ground_truth/\345\214\273\347\226\227.entities.json" "b/backend/evals/document_extraction/ground_truth/\345\214\273\347\226\227.entities.json" new file mode 100644 index 00000000..cb492fcb --- /dev/null +++ "b/backend/evals/document_extraction/ground_truth/\345\214\273\347\226\227.entities.json" @@ -0,0 +1,19 @@ +[ + "高血压", + "2型糖尿病", + "收缩压", + "舒张压", + "空腹血糖", + "糖化血红蛋白", + "肾功能", + "ACEI", + "ARB", + "CCB", + "二甲双胍", + "胰岛素", + "糖尿病肾病", + "视网膜病变", + "低血糖", + "药物相互作用", + "用药重整" +] diff --git "a/backend/evals/document_extraction/ground_truth/\350\264\242\345\212\241.entities.json" "b/backend/evals/document_extraction/ground_truth/\350\264\242\345\212\241.entities.json" new file mode 100644 index 00000000..2827b0ca --- /dev/null +++ "b/backend/evals/document_extraction/ground_truth/\350\264\242\345\212\241.entities.json" @@ -0,0 +1,17 @@ +[ + "供应商", + "发票", + "采购订单", + "采购审批", + "付款", + "费用报销", + "预算", + "预算调整", + "应收账款", + "账龄", + "坏账准备", + "固定资产", + "资产折旧", + "审计委员会", + "董事会" +] diff --git a/backend/evals/document_extraction/manifest/4domains.json b/backend/evals/document_extraction/manifest/4domains.json new file mode 100644 index 00000000..a6353945 --- /dev/null +++ b/backend/evals/document_extraction/manifest/4domains.json @@ -0,0 +1,6 @@ +[ + {"case_id": "credit-01", "domain": "信贷", "source_file": "test_data/信贷/信贷业务战略.md", "ground_truth_id": "信贷"}, + {"case_id": "supply-chain-01", "domain": "供应链", "source_file": "test_data/供应链/supply_chain_strategy.md", "ground_truth_id": "供应链"}, + {"case_id": "medical-01", "domain": "医疗", "source_file": "test_data/医疗/clinical_protocols.md", "ground_truth_id": "医疗"}, + {"case_id": "finance-01", "domain": "财务", "source_file": "test_data/财务/financial_controls.md", "ground_truth_id": "财务"} +] diff --git a/backend/evals/document_extraction/run.py b/backend/evals/document_extraction/run.py index 2a072e3a..6fd5f73b 100644 --- a/backend/evals/document_extraction/run.py +++ b/backend/evals/document_extraction/run.py @@ -4,7 +4,9 @@ import json import os -from evals.document_extraction.validators import dedup_gate, instance_leakage_gate, keyword_recall +from evals.document_extraction.validators import ( + dedup_gate, instance_leakage_gate, keyword_recall, precision_recall, rule_leakage_gate, +) def run_eval(manifest_path: str, ground_truth_dir: str, adapter, repo_root: str) -> dict: @@ -17,6 +19,15 @@ def run_eval(manifest_path: str, ground_truth_dir: str, adapter, repo_root: str) with open(gt_path, encoding="utf-8") as f: ground_truth = json.load(f) + # optional: a full gold entity list for real precision/recall/F1, + # separate from the keyword-hint ground_truth above — a domain + # without this file simply skips precision/recall, no breakage + entities_gt_path = os.path.join(ground_truth_dir, f"{case['ground_truth_id']}.entities.json") + gold_entities = None + if os.path.exists(entities_gt_path): + with open(entities_gt_path, encoding="utf-8") as f: + gold_entities = json.load(f) + source_path = os.path.join(repo_root, case["source_file"]) try: extraction_result = adapter.extract(source_path) @@ -26,7 +37,10 @@ def run_eval(manifest_path: str, ground_truth_dir: str, adapter, repo_root: str) "keyword_recall": keyword_recall(ground_truth, extraction_result), "dedup_findings": dedup_gate(extraction_result), "leakage_findings": instance_leakage_gate(extraction_result), + "rule_leakage_findings": rule_leakage_gate(extraction_result), } + if gold_entities is not None: + case_entry["precision_recall"] = precision_recall(gold_entities, extraction_result) except Exception as e: case_entry = { "case_id": case["case_id"], @@ -41,11 +55,16 @@ def run_eval(manifest_path: str, ground_truth_dir: str, adapter, repo_root: str) sum(c["keyword_recall"]["score"] for c in recall_cases) / len(recall_cases) if recall_cases else 0.0 ) + pr_cases = [c for c in case_results if "precision_recall" in c] + mean_f1 = sum(c["precision_recall"]["f1"] for c in pr_cases) / len(pr_cases) if pr_cases else None summary = { "total_cases": total, "mean_keyword_recall": mean_recall, "cases_with_dedup_findings": sum(1 for c in case_results if c.get("dedup_findings")), "cases_with_leakage_findings": sum(1 for c in case_results if c.get("leakage_findings")), + "cases_with_rule_leakage_findings": sum(1 for c in case_results if c.get("rule_leakage_findings")), + "cases_with_precision_recall": len(pr_cases), + "mean_f1": mean_f1, } return {"cases": case_results, "summary": summary} diff --git a/backend/evals/document_extraction/validators.py b/backend/evals/document_extraction/validators.py index c3b311fe..2e545313 100644 --- a/backend/evals/document_extraction/validators.py +++ b/backend/evals/document_extraction/validators.py @@ -13,6 +13,17 @@ _ID_PATTERN = re.compile(r"^[A-Za-z]+\d+$") _TRAILING_DIGITS_PATTERN = re.compile(r"\d{3,}$") +# A letter (+ optional 1-2 digits) followed by a small closed set of +# tier/stage/round markers, and nothing else — the shape of an enumeration +# VALUE of some concept's property (信用分层's "A层", 逾期阶段's "M2+", +# 融资轮次's "D轮融资"), not a concept in its own right. Kept in sync with +# app.services.llm_service._looks_like_enum_value. +_ENUM_VALUE_PATTERN = re.compile( + r"^[A-Za-z]{1,2}[+-]?(轮融资|阶段|层|级|档|类)$|^M\d{1,2}\+?(阶段)?$" +) + +_RULE_NAME_SUFFIXES = ("规则", "政策", "办法", "制度", "准则", "细则", "规程") + def _all_text_blobs(extraction_result: dict) -> list[str]: """Every name_cn/description/entity_type string across all categories, @@ -44,6 +55,51 @@ def keyword_recall(ground_truth: list[dict], extraction_result: dict) -> dict: return {"score": score, "passed": passed, "failed": failed} +def precision_recall(gold_entities: list[str], extraction_result: dict, fuzzy_threshold: int = 85) -> dict: + """Real precision/recall/F1 against a full gold-labeled entity list (not + keyword_recall's loose "does this keyword appear somewhere" check). + Fuzzy-matched (rapidfuzz, same threshold convention as dedup_gate) so a + minor surface-form difference between the gold name and the extracted + name_cn doesn't count as a miss — each gold entity and each extracted + entity is consumed by at most one match, so duplicates on either side + can't inflate the score.""" + extracted_names = [ + str(item.get("name_cn") or "") for item in (extraction_result.get("entities") or []) + if isinstance(item, dict) and item.get("name_cn") + ] + unmatched_extracted = list(extracted_names) + matched_gold: list[str] = [] + missed_gold: list[str] = [] + for gold_name in gold_entities: + best_idx, best_score = None, -1 + for idx, candidate in enumerate(unmatched_extracted): + score = fuzz.ratio(gold_name, candidate) + if score > best_score: + best_idx, best_score = idx, score + if best_idx is not None and best_score >= fuzzy_threshold: + matched_gold.append(gold_name) + unmatched_extracted.pop(best_idx) + else: + missed_gold.append(gold_name) + + true_positive = len(matched_gold) + false_negative = len(missed_gold) + false_positive = len(unmatched_extracted) # extracted but never matched a gold entity + precision = true_positive / (true_positive + false_positive) if (true_positive + false_positive) else 1.0 + recall = true_positive / (true_positive + false_negative) if (true_positive + false_negative) else 1.0 + f1 = (2 * precision * recall / (precision + recall)) if (precision + recall) else 0.0 + return { + "precision": round(precision, 3), + "recall": round(recall, 3), + "f1": round(f1, 3), + "true_positive": true_positive, + "false_positive": false_positive, + "false_negative": false_negative, + "missed_gold_entities": missed_gold, + "unmatched_extracted_entities": unmatched_extracted, + } + + def dedup_gate(extraction_result: dict, threshold: int = 85) -> list[dict]: findings: list[dict] = [] for category in ("entities", "logic_rules", "actions"): @@ -67,12 +123,33 @@ def _looks_like_instance_id(name: str) -> bool: return bool(_ID_PATTERN.match(name)) or bool(_TRAILING_DIGITS_PATTERN.search(name)) +def _looks_like_enum_value(name: str) -> bool: + return bool(_ENUM_VALUE_PATTERN.match(name)) + + def instance_leakage_gate(extraction_result: dict) -> list[dict]: findings: list[dict] = [] for item in extraction_result.get("entities") or []: if not isinstance(item, dict): continue name = str(item.get("name_cn") or "") - if name and _looks_like_instance_id(name): + if not name: + continue + if _looks_like_instance_id(name): findings.append({"name": name, "reason": "matches ID/numbered-record pattern"}) + elif _looks_like_enum_value(name): + findings.append({"name": name, "reason": "matches tier/stage/round enum-value pattern"}) + return findings + + +def rule_leakage_gate(extraction_result: dict) -> list[dict]: + """A business rule/policy name (e.g. "促销定价规则") is not an entity — + it belongs in logic_rules.""" + findings: list[dict] = [] + for item in extraction_result.get("entities") or []: + if not isinstance(item, dict): + continue + name = str(item.get("name_cn") or "") + if name and name.endswith(_RULE_NAME_SUFFIXES): + findings.append({"name": name, "reason": "matches business-rule/policy name pattern"}) return findings diff --git a/backend/openapi-agent.json b/backend/openapi-agent.json index c43d7de1..f0bfbdb2 100644 --- a/backend/openapi-agent.json +++ b/backend/openapi-agent.json @@ -14617,6 +14617,119 @@ ] } }, + "/api/v2/connections/{connection_id}/resources": { + "get": { + "description": "List tables/collections exposed by a saved connection.", + "operationId": "list_connection_resources_api_v2_connections__connection_id__resources_get", + "parameters": [ + { + "in": "path", + "name": "connection_id", + "required": true, + "schema": { + "title": "Connection Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "HTTPBearer": [] + } + ], + "summary": "List Connection Resources", + "tags": [ + "v2-connections" + ] + } + }, + "/api/v2/connections/{connection_id}/resources/{resource}/sample": { + "get": { + "description": "Read a bounded sample from one saved connection resource.", + "operationId": "sample_connection_resource_api_v2_connections__connection_id__resources__resource__sample_get", + "parameters": [ + { + "in": "path", + "name": "connection_id", + "required": true, + "schema": { + "title": "Connection Id", + "type": "string" + } + }, + { + "in": "path", + "name": "resource", + "required": true, + "schema": { + "title": "Resource", + "type": "string" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "default": 20, + "maximum": 1000, + "minimum": 1, + "title": "Limit", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "HTTPBearer": [] + } + ], + "summary": "Sample Connection Resource", + "tags": [ + "v2-connections" + ] + } + }, "/api/v2/connections/{connection_id}/schedule": { "post": { "description": "Compatibility delegate for the typed refresh schedule operation.", diff --git a/backend/pyproject.toml b/backend/pyproject.toml index c2826a3f..e41e1c44 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -2,7 +2,7 @@ name = "ontexus-backend" version = "0.1.0" description = "Ontexus backend service" -requires-python = ">=3.11" +requires-python = ">=3.11,<3.13" [tool.setuptools.packages.find] include = ["app*"] diff --git a/backend/scripts/check_python_version.py b/backend/scripts/check_python_version.py index eeec27e6..cf0fc6db 100644 --- a/backend/scripts/check_python_version.py +++ b/backend/scripts/check_python_version.py @@ -3,15 +3,18 @@ import sys MIN_SUPPORTED = (3, 11) +MAX_SUPPORTED_EXCLUSIVE = (3, 13) UNSUPPORTED_PYTHON_VERSION = "UNSUPPORTED_PYTHON_VERSION" def require_supported_python(): - if sys.version_info[:2] < MIN_SUPPORTED: + if sys.version_info[:2] < MIN_SUPPORTED or sys.version_info[:2] >= MAX_SUPPORTED_EXCLUSIVE: detected = ".".join(str(part) for part in sys.version_info[:3]) - required = ".".join(str(part) for part in MIN_SUPPORTED) + minimum = ".".join(str(part) for part in MIN_SUPPORTED) + maximum = ".".join(str(part) for part in MAX_SUPPORTED_EXCLUSIVE) raise SystemExit( - f"{UNSUPPORTED_PYTHON_VERSION}: detected Python {detected}, required >= {required}" + f"{UNSUPPORTED_PYTHON_VERSION}: detected Python {detected}, " + f"required >= {minimum}, < {maximum}" ) return True diff --git a/backend/tests/agent/test_core_route_registration.py b/backend/tests/agent/test_core_route_registration.py index ddbece72..f7f5f8eb 100644 --- a/backend/tests/agent/test_core_route_registration.py +++ b/backend/tests/agent/test_core_route_registration.py @@ -475,6 +475,8 @@ def test_idempotency_migration_upgrade_downgrade_cycle(): def test_audit_route_rejects_unknown_model_id_422(idem_schema): from app.routers.audit import router as audit_router from app.routers.audit import start_audit # noqa: F401 — route-local fix surface + from app.models.user import User + from app.services.ontology_access import creator_grant session_factory = sessionmaker(bind=create_engine(_scoped_url(idem_schema))) session = session_factory() @@ -491,6 +493,7 @@ def test_audit_route_rejects_unknown_model_id_422(idem_schema): "VALUES (:id, 'p', 'test', 'v0.1', 'created', :by, now(), now(), :domain, 1)" ), {"id": "onto-1", "by": admin_id, "domain": DEFAULT_DOMAIN}) session.commit() + creator_grant(session, "onto-1", session.get(User, admin_id)) finally: session.close() diff --git a/backend/tests/agent/test_extraction_normalization.py b/backend/tests/agent/test_extraction_normalization.py index 43ba1907..0edae9a7 100644 --- a/backend/tests/agent/test_extraction_normalization.py +++ b/backend/tests/agent/test_extraction_normalization.py @@ -15,10 +15,13 @@ from app.services.llm_service import ( VAGUE_RELATION_TYPES, + _looks_like_enum_value, + _looks_like_rule_name, apply_entity_resolution, extract_ontology, normalize_extracted_ontology, resolve_entities, + summarize_entity, ) BACKEND_DIR = Path(__file__).resolve().parents[2] @@ -39,7 +42,10 @@ def test_extraction_discipline_red_contract(): def test_normalization_dedups_and_drops_junk_over_test_data_document(): """A noisy extraction over the real 信贷 strategy document is normalized: duplicate surface forms merge, junk entities vanish, dangling and vague - relations are dropped, hierarchy relations survive.""" + relations are dropped, hierarchy relations survive. Enum-value labels + (tier/stage/round markers) and business-rule names are also entity + leakage — dropped, along with any relation left dangling by their + removal.""" document = TEST_DATA / "信贷" / "信贷业务战略.md" if not document.exists(): pytest.skip("test_data document unavailable") @@ -54,6 +60,7 @@ def test_normalization_dedups_and_drops_junk_over_test_data_document(): {"name_cn": "A层", "type": "Category", "description": "优质客户", "properties": {}}, {"name_cn": "B层", "type": "Category", "description": "良好客户", "properties": {}}, {"name_cn": "授信额度", "type": "Concept", "description": "可贷上限", "properties": {}}, + {"name_cn": "促销定价规则", "type": "Concept", "description": "误提取的规则", "properties": {}}, {"name_cn": "12345", "type": "Concept", "description": "噪音", "properties": {}}, {"name_cn": "", "type": "Concept", "description": "空名", "properties": {}}, {"name_cn": "信贷业务战略.md", "type": "Concept", "description": "文件名", "properties": {}}, @@ -85,6 +92,11 @@ def test_normalization_dedups_and_drops_junk_over_test_data_document(): # junk entities dropped assert "12345" not in names assert "信贷业务战略.md" not in names + # enum-value labels (tier markers) are not concepts — dropped + assert "A层" not in names + assert "B层" not in names + # a business rule is not an entity — dropped (belongs in logic_rules) + assert "促销定价规则" not in names assert not any(not (e.get("name_cn") or "").strip() for e in normalized["entities"]) # every surviving entity carries a grounded one-line description assert all(e.get("description") for e in normalized["entities"]) @@ -97,10 +109,31 @@ def test_normalization_dedups_and_drops_junk_over_test_data_document(): assert ("借款人", "DEPENDS_ON", "信用评分") in triples # the duplicate DEPENDS_ON appeared twice -> deduplicated to one assert sum(1 for r in normalized["relations"] if r["type"] == "DEPENDS_ON") == 1 - # hierarchy relations (IS-A / INSTANCE-OF) survive intact - assert ("A层", "INSTANCE-OF", "信用分层") in triples - assert ("B层", "INSTANCE-OF", "信用分层") in triples - assert ("A层", "GOVERNED_BY", "授信额度") in triples + # relations pointing at the dropped enum-value entities are now dangling + # references and are dropped too + assert ("A层", "INSTANCE-OF", "信用分层") not in triples + assert ("B层", "INSTANCE-OF", "信用分层") not in triples + assert ("A层", "GOVERNED_BY", "授信额度") not in triples + + +def test_looks_like_enum_value_matches_tier_stage_and_round_labels(): + for name in ("A层", "B层", "C级", "M0", "M1", "M2+", "M0阶段", "A轮融资", "D轮融资", "A档", "B类"): + assert _looks_like_enum_value(name), name + + +def test_looks_like_enum_value_does_not_match_real_concepts(): + for name in ("借款人", "供应商", "A股", "P7", "信用分层", "授信额度", "D轮"): + assert not _looks_like_enum_value(name), name + + +def test_looks_like_rule_name_matches_policy_and_rule_suffixes(): + for name in ("促销定价规则", "审批超时规则", "员工考勤制度", "风险准入政策"): + assert _looks_like_rule_name(name), name + + +def test_looks_like_rule_name_does_not_match_concepts(): + for name in ("借款人", "信用分层", "贷款产品"): + assert not _looks_like_rule_name(name), name def test_normalization_is_idempotent_and_keeps_clean_output(): @@ -208,6 +241,67 @@ def test_resolve_entities_single_type_member_skips_llm_call(): assert alias_map == {"唯一实体": "唯一实体"} +def test_resolve_entities_anchors_canonical_to_existing_entity(): + """Incremental-update guidance: a new-run surface form that clusters with + an already-published entity must remap to that entity's own name, not to + whatever the model picked as canonical.""" + from app.services import llm_service + original = llm_service._call_llm + + def fake_call(provider, api_key, api_base, model, messages): + assert "[已存在]" in messages[1]["content"] + return '{"clusters": [{"canonical": "客户方", "aliases": ["客户方", "客户"]}]}' + + llm_service._call_llm = fake_call + try: + new_entities = [{"name_cn": "客户方", "type": "Concept", "description": "本轮新提取"}] + existing = [{"name_cn": "客户", "type": "Concept", "description": "已发布的规范实体"}] + alias_map = resolve_entities( + new_entities, {"provider": "openai", "api_key": "x"}, "m", existing_entities=existing, + ) + finally: + llm_service._call_llm = original + # the model picked "客户方" as canonical, but the existing entity "客户" + # must win regardless + assert alias_map["客户方"] == "客户" + + +def test_resolve_entities_single_new_entity_still_checked_against_existing(): + """A single new entity (which would otherwise skip clustering entirely) + must still be checked against existing entities of the same type.""" + from app.services import llm_service + original = llm_service._call_llm + llm_service._call_llm = lambda *a, **k: ( + '{"clusters": [{"canonical": "借款人", "aliases": ["借款方", "借款人"]}]}' + ) + try: + alias_map = resolve_entities( + [{"name_cn": "借款方", "type": "Concept", "description": "本轮新提取"}], + {"provider": "openai", "api_key": "x"}, "m", + existing_entities=[{"name_cn": "借款人", "type": "Concept", "description": "已发布"}], + ) + finally: + llm_service._call_llm = original + assert alias_map["借款方"] == "借款人" + + +def test_resolve_entities_does_not_touch_existing_entities_of_other_types(): + """Existing entities of a different type never enter the same clustering + call — no cross-type contamination.""" + from app.services import llm_service + original = llm_service._call_llm + llm_service._call_llm = lambda *a, **k: (_ for _ in ()).throw(AssertionError("should not be called")) + try: + alias_map = resolve_entities( + [{"name_cn": "唯一实体", "type": "Concept", "description": "d"}], + {"provider": "openai", "api_key": "x"}, "m", + existing_entities=[{"name_cn": "供应商甲", "type": "Supplier", "description": "d"}], + ) + finally: + llm_service._call_llm = original + assert alias_map == {"唯一实体": "唯一实体"} + + def test_apply_entity_resolution_merges_and_remaps_references(): """apply_entity_resolution merges aliased entities (keeping the richer description), remaps relation endpoints, drops self-loops created by the @@ -255,3 +349,46 @@ def test_apply_entity_resolution_is_noop_for_identity_map(): result = {"entities": [{"name_cn": "A"}], "relations": []} assert apply_entity_resolution(result, {"A": "A"}) is result assert apply_entity_resolution(result, {}) is result + + +def test_summarize_entity_parses_profile_from_llm_response(): + from app.services import llm_service + original = llm_service._call_llm + llm_service._call_llm = lambda *a, **k: ( + '{"summary": "借款人是信贷业务的核心概念。", ' + '"key_facts": ["借款人需通过授信审批", "借款人拥有贷款账户", "多出来的第四条", ' + '"第五条", "第六条会被截断"], ' + '"time_range": {"start": "2026", "end": "ongoing"}}' + ) + try: + profile = summarize_entity( + "借款人", "贷款申请人", "(借款人) --[APPLIES_FOR]--> (贷款)", + {"provider": "openai", "api_key": "x"}, "m", + ) + finally: + llm_service._call_llm = original + assert profile["summary"] == "借款人是信贷业务的核心概念。" + assert len(profile["key_facts"]) == 5 # capped at 5 + assert profile["time_range"] == {"start": "2026", "end": "ongoing"} + + +def test_summarize_entity_returns_none_on_failure(): + from app.services import llm_service + original = llm_service._call_llm + llm_service._call_llm = lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom")) + try: + profile = summarize_entity("借款人", "d", "", {"provider": "openai", "api_key": "x"}, "m") + finally: + llm_service._call_llm = original + assert profile is None + + +def test_summarize_entity_defaults_missing_time_range_to_unknown(): + from app.services import llm_service + original = llm_service._call_llm + llm_service._call_llm = lambda *a, **k: '{"summary": "s", "key_facts": []}' + try: + profile = summarize_entity("X", "d", "", {"provider": "openai", "api_key": "x"}, "m") + finally: + llm_service._call_llm = original + assert profile["time_range"] == {"start": "unknown", "end": "unknown"} diff --git a/backend/tests/agent/test_import_mutation_closure.py b/backend/tests/agent/test_import_mutation_closure.py index a83d8e6d..42574636 100644 --- a/backend/tests/agent/test_import_mutation_closure.py +++ b/backend/tests/agent/test_import_mutation_closure.py @@ -133,8 +133,13 @@ def test_zz_extraction_rejects_unknown_model_with_stable_error(imports_db): from app.routers import extraction from app.services.auth_service import create_access_token + from app.services.ontology_access import backfill_creator_grants + Session, _ = imports_db actor_id, ontology_id = _seed(imports_db) + # _seed inserts via direct SQL, bypassing create_ontology's creator grant + with Session() as session: + backfill_creator_grants(session) token = create_access_token({"sub": actor_id, "role": "editor"}) headers = {"Authorization": f"Bearer {token}"} diff --git a/backend/tests/agent/test_legacy_route_closure.py b/backend/tests/agent/test_legacy_route_closure.py index be709588..b7bd9294 100644 --- a/backend/tests/agent/test_legacy_route_closure.py +++ b/backend/tests/agent/test_legacy_route_closure.py @@ -151,6 +151,8 @@ def closure_db(): def _seed(closure_db): from app.services.auth_service import hash_password + from app.models.user import User + from app.services.ontology_access import creator_grant Session, _ = closure_db username = "closure-" + uuid.uuid4().hex[:8] @@ -167,6 +169,7 @@ def _seed(closure_db): ), {"id": str(uuid.uuid4()), "name": "Closure " + uuid.uuid4().hex[:8], "creator": actor_id, "domain": DEFAULT_DOMAIN}) ontology_id = session.execute(text("SELECT id FROM ontology_projects WHERE created_by=:c ORDER BY created_at DESC LIMIT 1"), {"c": actor_id}).scalar_one() session.commit() + creator_grant(session, ontology_id, session.get(User, actor_id)) return actor_id, ontology_id diff --git a/backend/tests/agent/test_legacy_runtime_tables.py b/backend/tests/agent/test_legacy_runtime_tables.py index ae82eae0..577fc088 100644 --- a/backend/tests/agent/test_legacy_runtime_tables.py +++ b/backend/tests/agent/test_legacy_runtime_tables.py @@ -160,8 +160,11 @@ def test_audit_and_instance_writes_no_longer_500(legacy_schema): from app.deps import get_db from app.routers import audit as audit_router from app.services.auth_service import create_access_token + from app.services.ontology_access import backfill_creator_grants with Session() as session: + # the ontology was inserted via direct SQL, bypassing create_ontology's creator grant + backfill_creator_grants(session) actor_id = session.execute(text("SELECT id FROM users WHERE username='lr-user'")).scalar_one() token = create_access_token({"sub": actor_id, "role": "admin"}) diff --git a/backend/tests/agent/test_model_caller_cutover.py b/backend/tests/agent/test_model_caller_cutover.py index 07f075aa..3aedbc2f 100644 --- a/backend/tests/agent/test_model_caller_cutover.py +++ b/backend/tests/agent/test_model_caller_cutover.py @@ -178,6 +178,57 @@ def _headers(token): return {"Authorization": f"Bearer {token}"} +@pytest.mark.parametrize("reference", [None, "agent", "prompt"]) +def test_delete_model_preserves_migrated_reference_guards(pg, reference): + from fastapi import HTTPException + from app.models.agent import Agent, AgentVersion + from app.models.agent_config import PromptGeneration + from app.models.model_version import ModelMigrationFinding + from app.routers.models import delete_model + + model_id = _seed_llm_config(pg) + _migrate_rows(pg) + version_id = pg.execute(text( + "SELECT active_version_id FROM model_configs WHERE id = :id" + ), {"id": model_id}).scalar_one() + assert version_id is not None + pg.add(ModelMigrationFinding(model_config_id=model_id, code="TEST", reason="test finding")) + if reference: + pg.add(Agent(id="delete-test-agent", owner_id="seed-user")) + pg.flush() + if reference == "agent": + schema_id = pg.execute(text( + "SELECT active_version_id FROM application_state_schema_registries WHERE application_key = 'chat-v1'" + )).scalar_one() + pg.add(AgentVersion( + agent_id="delete-test-agent", version_no=1, name="dependent", + default_model_config_version_id=version_id, default_model_name="gpt-4o", + application_state_schema_version_id=schema_id, config_hash="a" * 64, + created_by="seed-user", + )) + else: + pg.add(PromptGeneration( + agent_id="delete-test-agent", base_version_no=1, + model_config_version_id=version_id, model_name="gpt-4o", + input_hash="b" * 64, output_hash="c" * 64, requester_id="seed-user", + )) + pg.commit() + + for table in ("model_config_versions", "model_credentials", "model_migration_findings"): + assert pg.execute(text(f"SELECT count(*) FROM {table} WHERE model_config_id = :id"), {"id": model_id}).scalar_one() > 0 + if reference: + with pytest.raises(HTTPException) as denied: + delete_model(model_id, db=pg) + assert denied.value.status_code == 409 + assert denied.value.detail == "MODEL_REFERENCED" + assert pg.execute(text("SELECT active_version_id FROM model_configs WHERE id = :id"), {"id": model_id}).scalar_one() == version_id + else: + delete_model(model_id, db=pg) + assert pg.execute(text("SELECT count(*) FROM model_configs WHERE id = :id"), {"id": model_id}).scalar_one() == 0 + for table in ("model_config_versions", "model_credentials", "model_migration_findings"): + assert pg.execute(text(f"SELECT count(*) FROM {table} WHERE model_config_id = :id"), {"id": model_id}).scalar_one() == 0 + + def test_port_resolves_immutable_llm_version_and_rejects_blocked(pg): from app.services.publication.extraction_model_port import SqlExtractionModelPort @@ -277,8 +328,10 @@ def test_model_api_redacted_tagged_union_and_versioned_surface(pg): assert detail["active_version"]["version_no"] == 2 assert detail["active_version"]["behavior_hash"] != versions[0]["behavior_hash"] - # delete is RESTRICT while versions exist - assert client.delete(f"/api/v1/models/{llm_id}", headers=headers).status_code == 409 + # Owned versions alone do not make an unused model undeletable. Real + # Agent/prompt references are covered separately by the deletion matrix. + assert client.delete(f"/api/v1/models/{llm_id}", headers=headers).status_code == 204 + assert client.get(f"/api/v1/models/{llm_id}", headers=headers).status_code == 404 # viewer cannot create/update models (editor ceiling) assert client.post("/api/v1/models", json={"name": "x", "provider": "openai", "models": ["gpt-4o"]}, headers=viewer_headers).status_code == 403 diff --git a/backend/tests/agent/test_ontology_access_admin.py b/backend/tests/agent/test_ontology_access_admin.py index 02a65eec..d7f7b077 100644 --- a/backend/tests/agent/test_ontology_access_admin.py +++ b/backend/tests/agent/test_ontology_access_admin.py @@ -11,6 +11,7 @@ import hashlib import json import os +from datetime import datetime, timedelta, timezone from pathlib import Path import subprocess import sys @@ -79,7 +80,7 @@ def access_db(): engine.dispose() -def _insert_user(Session, username, role="editor", active=True, domain=True): +def _insert_user(Session, username, role="editor", active=True, domain=True, domain_id=None): domain_clause = ", security_domain_id" if domain else "" domain_value = ", :domain" if domain else "" params = { @@ -90,7 +91,7 @@ def _insert_user(Session, username, role="editor", active=True, domain=True): "active": active, } if domain: - params["domain"] = DEFAULT_DOMAIN + params["domain"] = domain_id or DEFAULT_DOMAIN with Session() as session: session.execute(text( f"INSERT INTO users (id, username, email, password_hash, role, is_active, created_at, updated_at{domain_clause}) " @@ -101,12 +102,12 @@ def _insert_user(Session, username, role="editor", active=True, domain=True): return user_id -def _insert_ontology(Session, name, created_by): +def _insert_ontology(Session, name, created_by, domain_id=None): with Session() as session: session.execute(text( "INSERT INTO ontology_projects (id, name, domain, version, status, created_by, created_at, updated_at, security_domain_id) " "VALUES (:id, :name, 'test', 'v0.1', 'draft', :created_by, now(), now(), :domain)" - ), {"id": str(uuid.uuid4()), "name": name, "created_by": created_by, "domain": DEFAULT_DOMAIN}) + ), {"id": str(uuid.uuid4()), "name": name, "created_by": created_by, "domain": domain_id or DEFAULT_DOMAIN}) ontology_id = session.execute(text("SELECT id FROM ontology_projects WHERE name=:n"), {"n": name}).scalar_one() session.commit() return ontology_id @@ -501,3 +502,166 @@ def override_get_db(): ) assert missing.status_code == 404 test_app.dependency_overrides.clear() + + +def test_zz_require_project_grant_enforces_actor_state_domain_window_and_role(access_db): + from types import SimpleNamespace + + from app.models.user import User + from app.services.ontology_access import ( + GrantDenied, + create_grant, + creator_grant, + require_project_grant, + ) + + Session, _ = access_db + owner = _insert_user(Session, "guard-owner", role="admin") + reader = _insert_user(Session, "guard-reader", role="editor") + target = _insert_user(Session, "guard-target", role="viewer") + ontology_id = _insert_ontology(Session, "Guard ontology", owner) + + with Session() as session: + creator_grant(session, ontology_id, session.get(User, owner)) + create_grant( + session, + ontology_id=ontology_id, + user_id=reader, + capabilities=["discover", "read", "edit", "publish"], + base_revision=0, + actor_id=owner, + ) + reader_user = session.get(User, reader) + assert require_project_grant(session, reader_user, ontology_id, "edit") + + reader_user.role = "viewer" + session.flush() + with pytest.raises(GrantDenied): + require_project_grant(session, reader_user, ontology_id, "edit") + assert require_project_grant(session, reader_user, ontology_id, "read") + + reader_user.role = "editor" + session.execute(text( + "UPDATE ontology_project_access_grants SET valid_from=:valid_from " + "WHERE ontology_id=:o AND user_id=:u" + ), { + "valid_from": datetime.now(timezone.utc) + timedelta(minutes=5), + "o": ontology_id, + "u": reader, + }) + with pytest.raises(GrantDenied): + require_project_grant(session, reader_user, ontology_id, "read") + + session.execute(text( + "UPDATE ontology_project_access_grants SET valid_from=NULL, valid_until=:valid_until " + "WHERE ontology_id=:o AND user_id=:u" + ), { + "valid_until": datetime.now(timezone.utc) - timedelta(seconds=1), + "o": ontology_id, + "u": reader, + }) + with pytest.raises(GrantDenied): + require_project_grant(session, reader_user, ontology_id, "read") + + session.execute(text( + "UPDATE ontology_project_access_grants SET valid_until=NULL WHERE ontology_id=:o AND user_id=:u" + ), {"o": ontology_id, "u": reader}) + with pytest.raises(GrantDenied): + require_project_grant( + session, + SimpleNamespace( + id=reader, + is_active=True, + role="editor", + security_domain_id="00000000-0000-0000-0000-000000000099", + ), + ontology_id, + "read", + ) + + reader_user.is_active = False + session.flush() + with pytest.raises(GrantDenied): + require_project_grant(session, reader_user, ontology_id, "read") + + session.execute(text( + "UPDATE ontology_project_access_grants SET valid_until=:valid_until " + "WHERE ontology_id=:o AND user_id=:u" + ), { + "valid_until": datetime.now(timezone.utc) - timedelta(seconds=1), + "o": ontology_id, + "u": owner, + }) + with pytest.raises(GrantDenied): + create_grant( + session, + ontology_id=ontology_id, + user_id=target, + capabilities=["discover", "read"], + base_revision=0, + actor_id=owner, + ) + + session.execute(text( + "UPDATE ontology_project_access_grants SET valid_until=NULL " + "WHERE ontology_id=:o AND user_id=:u" + ), {"o": ontology_id, "u": owner}) + session.execute(text("UPDATE users SET role='viewer' WHERE id=:u"), {"u": owner}) + with pytest.raises(GrantDenied): + create_grant( + session, + ontology_id=ontology_id, + user_id=target, + capabilities=["discover", "read"], + base_revision=0, + actor_id=owner, + ) + + +def test_zz_discoverable_ontology_ids_filter_capability_window_domain_and_role(access_db): + from app.models.user import User + from app.services.ontology_access import ( + create_grant, + creator_grant, + discoverable_ontology_ids, + ) + + Session, _ = access_db + owner = _insert_user(Session, "discover-owner", role="admin") + viewer = _insert_user(Session, "discover-viewer", role="viewer") + same = _insert_ontology(Session, "Discover same", owner) + no_discover = _insert_ontology(Session, "Discover no capability", owner) + expired = _insert_ontology(Session, "Discover expired", owner) + future = _insert_ontology(Session, "Discover future", owner) + + with Session() as session: + creator_grant(session, same, session.get(User, owner)) + creator_grant(session, no_discover, session.get(User, owner)) + creator_grant(session, expired, session.get(User, owner)) + creator_grant(session, future, session.get(User, owner)) + for oid, capabilities in ( + (same, ["discover", "read"]), + (no_discover, ["read"]), + (expired, ["discover", "read"]), + (future, ["discover", "read"]), + ): + create_grant( + session, + ontology_id=oid, + user_id=viewer, + capabilities=capabilities, + base_revision=0, + actor_id=owner, + ) + session.execute(text( + "UPDATE ontology_project_access_grants SET valid_until=:when " + "WHERE ontology_id=:o AND user_id=:u" + ), {"when": datetime.now(timezone.utc) - timedelta(seconds=1), "o": expired, "u": viewer}) + session.execute(text( + "UPDATE ontology_project_access_grants SET valid_from=:when " + "WHERE ontology_id=:o AND user_id=:u" + ), {"when": datetime.now(timezone.utc) + timedelta(minutes=5), "o": future, "u": viewer}) + actor = session.get(User, viewer) + ids = session.execute(discoverable_ontology_ids(session, actor)).scalars().all() + + assert ids == [same] diff --git a/backend/tests/agent/test_ontology_route_access.py b/backend/tests/agent/test_ontology_route_access.py new file mode 100644 index 00000000..6eda7805 --- /dev/null +++ b/backend/tests/agent/test_ontology_route_access.py @@ -0,0 +1,80 @@ +"""Project grants must protect legacy and classic-v2 ontology API surfaces.""" + +import pytest + +from app.main import app +from app.routers.v2 import graph as graph_v2 +from app.routers.v2 import logic_actions as logic_actions_v2 +from app.routers.v2 import search as search_v2 + + +@pytest.mark.parametrize("path", [ + "/api/v1/ontologies/{id}", + "/api/v1/ontologies/{id}/export?format=json", + "/api/v1/ontologies/{id}/entities", + "/api/v1/ontologies/{id}/logic", + "/api/v1/ontologies/{id}/actions", + "/api/v1/ontologies/{id}/graph", + "/api/v1/ontologies/{id}/files", + "/api/v2/ontologies/{id}/graph", + "/api/v2/ontologies/{id}/search/keyword?q=test", + "/api/v2/ontologies/{id}/logic", + "/api/v2/ontologies/{id}/state-machines", + "/api/v2/ontologies/{id}/actions", +]) +def test_ungranted_editor_cannot_read_ontology_surfaces(client, db, ontology, editor_user, path): + def test_db(): + yield db + + for dependency in (graph_v2.get_db, search_v2.get_db, logic_actions_v2.get_db): + app.dependency_overrides[dependency] = test_db + + token = client.post( + "/api/v1/auth/login", + json={"username": editor_user.username, "password": "editor123"}, + ).json()["data"]["access_token"] + response = client.get( + path.format(id=ontology["id"]), + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 404 + + +@pytest.mark.parametrize("path", [ + "/api/v1/ontologies/{id}/export?format=json", + "/api/v2/ontologies/{id}/search/keyword?q=test", + "/api/v2/ontologies/{id}/logic", +]) +def test_creator_grant_keeps_read_routes_available(client, db, ontology, auth_headers, path): + def test_db(): + yield db + + for dependency in (search_v2.get_db, logic_actions_v2.get_db): + app.dependency_overrides[dependency] = test_db + + response = client.get(path.format(id=ontology["id"]), headers=auth_headers) + assert response.status_code == 200 + + +def _editor_headers(client): + token = client.post( + "/api/v1/auth/login", + json={"username": "editor", "password": "editor123"}, + ).json()["data"]["access_token"] + return {"Authorization": f"Bearer {token}"} + + +def test_ungranted_editor_cannot_write_legacy_entities(client, ontology, editor_user): + response = client.post( + f"/api/v1/ontologies/{ontology['id']}/entities", + json={"name_cn": "越权实体", "name_en": "Intruder"}, + headers=_editor_headers(client), + ) + assert response.status_code == 404 + + +def test_overview_hides_ungranted_ontologies(client, ontology, editor_user): + response = client.get("/api/v1/overview/stats", headers=_editor_headers(client)) + assert response.status_code == 200 + data = response.json()["data"] + assert ontology["id"] not in [item["id"] for item in data["recent_ontologies"]] diff --git a/backend/tests/agent/test_python_floor.py b/backend/tests/agent/test_python_floor.py index 2ec9b4e9..595336d4 100644 --- a/backend/tests/agent/test_python_floor.py +++ b/backend/tests/agent/test_python_floor.py @@ -14,7 +14,7 @@ BOOTSTRAP_PATH = BACKEND_DIR / "scripts" / "bootstrap_backend.py" PYPROJECT_PATH = BACKEND_DIR / "pyproject.toml" UNSUPPORTED_PYTHON_VERSION = "UNSUPPORTED_PYTHON_VERSION" -REQUIRED_PYTHON_FLOOR = ">=3.11" +REQUIRED_PYTHON_RANGE = ">=3.11,<3.13" def _run_entrypoint(*args, env=None): @@ -59,10 +59,10 @@ def test_e0_py_red_contract(): if missing: pytest.fail("RED_E0_PY: Python floor contract missing: " + ", ".join(missing)) metadata = tomllib.loads(PYPROJECT_PATH.read_text()) - if metadata.get("project", {}).get("requires-python") != REQUIRED_PYTHON_FLOOR: + if metadata.get("project", {}).get("requires-python") != REQUIRED_PYTHON_RANGE: pytest.fail( f"RED_E0_PY: pyproject.toml must declare requires-python = " - f"{REQUIRED_PYTHON_FLOOR!r}" + f"{REQUIRED_PYTHON_RANGE!r}" ) proc = _run_guard() if proc.returncode != 0: @@ -207,8 +207,8 @@ def test_require_supported_python_accepts_current_runtime(): assert _load_guard().require_supported_python() is True -def test_require_supported_python_rejects_python_3_10_and_earlier(): - for major_minor in [(3, 10), (3, 9), (3, 8)]: +def test_require_supported_python_rejects_versions_outside_supported_range(): + for major_minor in [(3, 10), (3, 9), (3, 8), (3, 13)]: proc = subprocess.run( [ sys.executable, @@ -225,7 +225,7 @@ def test_require_supported_python_rejects_python_3_10_and_earlier(): assert UNSUPPORTED_PYTHON_VERSION in message detected = ".".join(str(part) for part in major_minor) assert f"detected Python {detected}" in message - assert "required >= 3.11" in message + assert "required >= 3.11, < 3.13" in message def test_guard_runs_without_site_packages(): @@ -248,7 +248,7 @@ def test_guard_imports_stdlib_only(): def test_pyproject_declares_python_floor(): metadata = tomllib.loads(PYPROJECT_PATH.read_text()) - assert metadata["project"]["requires-python"] == REQUIRED_PYTHON_FLOOR + assert metadata["project"]["requires-python"] == REQUIRED_PYTHON_RANGE def test_packaging_build_evaluates_and_emits_python_floor(tmp_path): @@ -278,4 +278,4 @@ def test_packaging_build_evaluates_and_emits_python_floor(tmp_path): assert len(installed) == 1 metadata = installed[0]["metadata"] assert metadata["name"] == "ontexus-backend" - assert metadata["requires_python"] == REQUIRED_PYTHON_FLOOR + assert set(metadata["requires_python"].split(",")) == set(REQUIRED_PYTHON_RANGE.split(",")) diff --git a/backend/tests/agent/test_simple_llm_instances.py b/backend/tests/agent/test_simple_llm_instances.py index 7b1333f5..f5d66094 100644 --- a/backend/tests/agent/test_simple_llm_instances.py +++ b/backend/tests/agent/test_simple_llm_instances.py @@ -4,6 +4,7 @@ instance data attached to its concept entities (row_identity + row_data), consistent with Pipeline/Mapping output — not just Entity/Relation rows. """ +import json import os import subprocess import sys @@ -162,6 +163,164 @@ def test_simple_llm_tabular_extraction_materializes_instances(schema, monkeypatc session.close() +def test_instance_relations_reuses_relation_from_plain_relations_array(schema, monkeypatch): + """Bug caught live on a 供应链 rebuild: when the LLM's own `relations` array + (not the instance_relations auto-create branch) is what creates the + concept-level Relation that an instance_relations entry then reuses via + relation_id_by_key, that relation must still be flushed before the + EntityInstanceRelation insert — otherwise it's an unflushed pending row + and the FK insert fails. Production's SessionLocal is autoflush=False + (app/database.py), unlike _worker's default-autoflush factory, so this + test builds its own session to match and actually catch the bug.""" + factory = sessionmaker(bind=create_engine(_scoped_url(schema)), autoflush=False) + session = factory() + task_id = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + + fake_result = { + "entities": [ + {"name_cn": "供应商", "type": "Supplier", "description": "d", "properties": {}}, + {"name_cn": "物料", "type": "Material", "description": "d", "properties": {}}, + ], + "relations": [ + {"source": "供应商", "target": "物料", "type": "supply", "confidence": 0.9}, + ], + "logic_rules": [], "actions": [], + "instances": [ + {"entity_type": "供应商", "name_cn": "天钢原材料有限公司", "properties": {}}, + {"entity_type": "物料", "name_cn": "钢材", "properties": {}}, + ], + "instance_relations": [ + {"source": "天钢原材料有限公司", "target": "钢材", "type": "supply", "confidence": 0.9}, + ], + } + monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: fake_result) + + extraction_task.run_extraction(task_id) + try: + status = session.execute(text( + "SELECT status, error FROM extraction_tasks WHERE id = :id" + ), {"id": task_id}).mappings().one() + assert status["status"] == "completed", status["error"] + + edge = session.execute(text( + "SELECT ei_src.row_identity AS src, ei_tgt.row_identity AS tgt " + "FROM entity_instance_relations eir " + "JOIN entity_instances ei_src ON ei_src.id = eir.source_instance_id " + "JOIN entity_instances ei_tgt ON ei_tgt.id = eir.target_instance_id " + "WHERE eir.ontology_id = 'o-llm'" + )).mappings().one() + assert edge["src"] == "天钢原材料有限公司" + assert edge["tgt"] == "钢材" + finally: + session.close() + + +def test_instance_relations_create_concept_relation_and_instance_edge(schema, monkeypatch): + """A causal/interaction relation between two SPECIFIC named instances + (e.g. 2型糖尿病 causes 糖尿病肾病) must not be silently dropped just + because 'relations only connect concepts' — it auto-creates the concept- + level Relation it instantiates and lands as an EntityInstanceRelation + edge between the two instance rows. Uses an explicit autoflush=False + session to match production (app/database.py), not _worker's default.""" + factory = sessionmaker(bind=create_engine(_scoped_url(schema)), autoflush=False) + session = factory() + task_id = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + + fake_result = { + "entities": [{"name_cn": "疾病", "type": "Disease", "description": "d", "properties": {}}], + "relations": [], "logic_rules": [], "actions": [], + "instances": [ + {"entity_type": "疾病", "name_cn": "2型糖尿病", "properties": {}}, + {"entity_type": "疾病", "name_cn": "糖尿病肾病", "properties": {}}, + ], + "instance_relations": [ + {"source": "2型糖尿病", "target": "糖尿病肾病", "type": "causes", "confidence": 0.9}, + ], + } + monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: fake_result) + + extraction_task.run_extraction(task_id) + try: + status = session.execute(text( + "SELECT status, error FROM extraction_tasks WHERE id = :id" + ), {"id": task_id}).mappings().one() + assert status["status"] == "completed", status["error"] + + concept_rels = session.execute(text( + "SELECT type FROM relations WHERE ontology_id = 'o-llm'" + )).mappings().all() + assert [r["type"] for r in concept_rels] == ["causes"] + + edge = session.execute(text( + "SELECT ei_src.row_identity AS src, ei_tgt.row_identity AS tgt " + "FROM entity_instance_relations eir " + "JOIN entity_instances ei_src ON ei_src.id = eir.source_instance_id " + "JOIN entity_instances ei_tgt ON ei_tgt.id = eir.target_instance_id " + "WHERE eir.ontology_id = 'o-llm'" + )).mappings().one() + assert edge["src"] == "2型糖尿病" + assert edge["tgt"] == "糖尿病肾病" + finally: + session.close() + + +def test_instance_relations_rerun_does_not_duplicate(schema, monkeypatch): + """Re-running extraction with the same instance_relations output must not + create a second EntityInstanceRelation edge or a second concept Relation. + Uses an explicit autoflush=False session to match production.""" + factory = sessionmaker(bind=create_engine(_scoped_url(schema)), autoflush=False) + session = factory() + task_id_1 = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + + fake_result = { + "entities": [{"name_cn": "疾病", "type": "Disease", "description": "d", "properties": {}}], + "relations": [], "logic_rules": [], "actions": [], + "instances": [ + {"entity_type": "疾病", "name_cn": "2型糖尿病", "properties": {}}, + {"entity_type": "疾病", "name_cn": "糖尿病肾病", "properties": {}}, + ], + "instance_relations": [ + {"source": "2型糖尿病", "target": "糖尿病肾病", "type": "causes", "confidence": 0.9}, + ], + } + monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: fake_result) + extraction_task.run_extraction(task_id_1) + + task_id_2 = str(uuid.uuid4()) + session.execute(text( + "INSERT INTO extraction_tasks (id, ontology_id, prompt_id, model_id, status, parameters, progress, error, created_at, updated_at) " + "VALUES (:id, 'o-llm', 'p-1', 'm-1', 'queued', :params, '{}'::json, NULL, now(), now())" + ), {"id": task_id_2, "params": '{"model_name": "mock-extractor", "constraints": []}'}) + session.commit() + monkeypatch.setattr( + "app.services.llm_service._call_llm", + lambda *a, **k: '{"clusters": []}', + ) + extraction_task.run_extraction(task_id_2) + + try: + status = session.execute(text( + "SELECT status, error FROM extraction_tasks WHERE id = :id" + ), {"id": task_id_2}).mappings().one() + assert status["status"] == "completed", status["error"] + rel_count = session.execute(text( + "SELECT count(*) FROM relations WHERE ontology_id = 'o-llm'" + )).scalar_one() + edge_count = session.execute(text( + "SELECT count(*) FROM entity_instance_relations WHERE ontology_id = 'o-llm'" + )).scalar_one() + assert rel_count == 1 + assert edge_count == 1 + finally: + session.close() + + def test_single_document_near_duplicate_entities_are_resolved(schema, monkeypatch): """Sub-project E3 bug 1: a single-file extraction with near-duplicate entity names (e.g. "产品研发部" vs "产品研发部门") must still go through @@ -186,7 +345,7 @@ def test_single_document_near_duplicate_entities_are_resolved(schema, monkeypatc ) monkeypatch.setattr( "app.services.llm_service.resolve_entities", - lambda entities, config, model_name: { + lambda entities, config, model_name, existing_entities=None: { "产品研发部": "产品研发部", "产品研发部门": "产品研发部", }, @@ -237,3 +396,267 @@ def test_simple_llm_without_instances_still_succeeds(schema, monkeypatch): )).scalar_one() == 0 finally: session.close() + + +def test_dangling_linked_entities_reference_is_dropped(schema, monkeypatch): + """A logic_rule/action whose linked_entities names a rule-category label + (e.g. "反欺诈规则") rather than a real concept entity must not carry that + dangling reference into the saved row — the same discipline already + applied to relations pointing at an entity the model never extracted.""" + factory, session = _worker(schema) + task_id = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + monkeypatch.setattr( + "app.services.llm_service.extract_ontology", + lambda *a, **k: { + "entities": [{"name_cn": "设备指纹", "type": "Concept", "description": "设备指纹特征", "properties": {}}], + "relations": [], + "logic_rules": [ + {"name_cn": "设备指纹黑名单拒绝", "function_type": "derived_property", "definition": "命中黑名单则拒绝", + "linked_entities": ["设备指纹", "反欺诈规则"]}, + ], + "actions": [ + {"name_cn": "标记欺诈", "rules": [], "linked_entities": ["设备指纹", "反欺诈规则"]}, + ], + }, + ) + extraction_task.run_extraction(task_id) + try: + status = session.execute(text( + "SELECT status, error FROM extraction_tasks WHERE id = :id" + ), {"id": task_id}).mappings().one() + assert status["status"] == "completed", status["error"] + def _as_list(value): + return json.loads(value) if isinstance(value, str) else value + + rule_linked = _as_list(session.execute(text( + "SELECT linked_entities FROM logic_rules WHERE ontology_id = 'o-llm' AND name_cn = '设备指纹黑名单拒绝'" + )).scalar_one()) + assert rule_linked == ["设备指纹"] + action_linked = _as_list(session.execute(text( + "SELECT linked_entities FROM actions WHERE ontology_id = 'o-llm' AND name_cn = '标记欺诈'" + )).scalar_one()) + assert action_linked == ["设备指纹"] + finally: + session.close() + + +def test_rerun_resolves_new_surface_form_against_already_published_entity(schema, monkeypatch): + """Incremental-update guidance: re-running extraction on an ontology that + already has a published entity must not create a duplicate when the new + run's LLM output spells the same concept differently — it should resolve + to the existing entity's own name, not the other way around.""" + factory, session = _worker(schema) + task_id_1 = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + + monkeypatch.setattr( + "app.services.llm_service.extract_ontology", + lambda *a, **k: { + "entities": [{"name_cn": "客户", "type": "Concept", "description": "贷款客户", "properties": {}}], + "relations": [], "logic_rules": [], "actions": [], + }, + ) + extraction_task.run_extraction(task_id_1) + + task_id_2 = str(uuid.uuid4()) + session.execute(text( + "INSERT INTO extraction_tasks (id, ontology_id, prompt_id, model_id, status, parameters, progress, error, created_at, updated_at) " + "VALUES (:id, 'o-llm', 'p-1', 'm-1', 'queued', :params, '{}'::json, NULL, now(), now())" + ), {"id": task_id_2, "params": '{"model_name": "mock-extractor", "constraints": []}'}) + session.commit() + + monkeypatch.setattr( + "app.services.llm_service.extract_ontology", + lambda *a, **k: { + "entities": [{"name_cn": "客户方", "type": "Concept", "description": "本轮换了个写法", "properties": {}}], + "relations": [], "logic_rules": [], "actions": [], + }, + ) + monkeypatch.setattr( + "app.services.llm_service._call_llm", + lambda *a, **k: '{"clusters": [{"canonical": "客户方", "aliases": ["客户方", "客户"]}]}', + ) + extraction_task.run_extraction(task_id_2) + + try: + status = session.execute(text( + "SELECT status, error FROM extraction_tasks WHERE id = :id" + ), {"id": task_id_2}).mappings().one() + assert status["status"] == "completed", status["error"] + names = session.execute(text( + "SELECT name_cn FROM entities WHERE ontology_id = 'o-llm'" + )).scalars().all() + # the model's clustering call picked "客户方" as canonical, but the + # already-published "客户" must win — no duplicate, no rename + assert names == ["客户"] + finally: + session.close() + + +def test_extraction_task_stores_graph_diagnostics(schema, monkeypatch): + """Graph diagnostics (connected components, hub entities) are computed + over the ontology's full saved graph and stored on the task's + validation_report alongside the existing P0 report.""" + factory, session = _worker(schema) + task_id = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + monkeypatch.setattr( + "app.services.llm_service.extract_ontology", + lambda *a, **k: { + "entities": [ + {"name_cn": "借款人", "type": "Concept", "description": "d", "properties": {}}, + {"name_cn": "贷款", "type": "Concept", "description": "d", "properties": {}}, + {"name_cn": "孤立实体", "type": "Concept", "description": "d", "properties": {}}, + ], + "relations": [{"source": "借款人", "target": "贷款", "type": "APPLIES_FOR", "confidence": 0.9}], + "logic_rules": [], "actions": [], + }, + ) + extraction_task.run_extraction(task_id) + try: + report = session.execute(text( + "SELECT validation_report FROM extraction_tasks WHERE id = :id" + ), {"id": task_id}).scalar_one() + diagnostics = json.loads(report)["graph_diagnostics"] if isinstance(report, str) else report["graph_diagnostics"] + assert diagnostics["entity_count"] == 3 + assert diagnostics["edge_count"] == 1 + assert diagnostics["connected_components"] == 2 # {借款人,贷款} + {孤立实体} + assert diagnostics["isolated_entity_count"] == 1 + assert diagnostics["isolated_entities"] == ["孤立实体"] + finally: + session.close() + + +def test_graph_diagnostics_sees_this_runs_own_relations_under_autoflush_off(schema, monkeypatch): + """Production's SessionLocal is autoflush=False (app/database.py), unlike + the _worker fixture's default-autoflush factory used above. The + diagnostics query must still see this run's own newly-added entities and + relations rather than only whatever was committed before this run began.""" + factory = sessionmaker(bind=create_engine(_scoped_url(schema)), autoflush=False) + session = factory() + task_id = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + monkeypatch.setattr( + "app.services.llm_service.extract_ontology", + lambda *a, **k: { + "entities": [ + {"name_cn": "借款人", "type": "Concept", "description": "d", "properties": {}}, + {"name_cn": "贷款", "type": "Concept", "description": "d", "properties": {}}, + ], + "relations": [{"source": "借款人", "target": "贷款", "type": "APPLIES_FOR", "confidence": 0.9}], + "logic_rules": [], "actions": [], + }, + ) + extraction_task.run_extraction(task_id) + try: + report = session.execute(text( + "SELECT validation_report FROM extraction_tasks WHERE id = :id" + ), {"id": task_id}).scalar_one() + diagnostics = json.loads(report)["graph_diagnostics"] if isinstance(report, str) else report["graph_diagnostics"] + assert diagnostics["edge_count"] == 1 + assert diagnostics["connected_components"] == 1 + finally: + session.close() + + +_HUB_RESULT = { + "entities": [ + {"name_cn": "借款人", "type": "Concept", "description": "贷款申请人", "properties": {}}, + {"name_cn": "贷款", "type": "Concept", "description": "d", "properties": {}}, + {"name_cn": "授信额度", "type": "Concept", "description": "d", "properties": {}}, + {"name_cn": "信用评分", "type": "Concept", "description": "d", "properties": {}}, + ], + "relations": [ + {"source": "借款人", "target": "贷款", "type": "APPLIES_FOR", "confidence": 0.9}, + {"source": "借款人", "target": "授信额度", "type": "HAS", "confidence": 0.9}, + {"source": "借款人", "target": "信用评分", "type": "HAS", "confidence": 0.9}, + ], + "logic_rules": [], "actions": [], +} + + +def test_hub_entity_gets_summarized(schema, monkeypatch): + """A degree-3+ entity (playbook Section V.A hub threshold) gets a + synthesized profile stored on its properties; low-degree entities do not.""" + factory, session = _worker(schema) + task_id = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: _HUB_RESULT) + monkeypatch.setattr( + "app.services.llm_service._call_llm", + lambda *a, **k: '{"summary": "借款人是核心概念。", "key_facts": ["f1"], ' + '"time_range": {"start": "2026", "end": "ongoing"}}', + ) + extraction_task.run_extraction(task_id) + try: + status = session.execute(text( + "SELECT status, error FROM extraction_tasks WHERE id = :id" + ), {"id": task_id}).mappings().one() + assert status["status"] == "completed", status["error"] + rows = session.execute(text( + "SELECT name_cn, properties FROM entities WHERE ontology_id = 'o-llm'" + )).mappings().all() + props_by_name = {r["name_cn"]: (json.loads(r["properties"]) if isinstance(r["properties"], str) else r["properties"]) for r in rows} + assert props_by_name["借款人"]["summary"] == "借款人是核心概念。" + assert props_by_name["借款人"]["summary_degree"] == 3 + # a degree-1 entity is not summarized + assert "summary" not in props_by_name["贷款"] + finally: + session.close() + + +def test_hub_entity_summary_skipped_when_degree_unchanged_on_rerun(schema, monkeypatch): + """Re-running with the same graph shape must not re-call the summarizer — + only a changed source-document set (reflected here as a changed degree) + should trigger re-summarization.""" + factory, session = _worker(schema) + task_id_1 = _seed(session) + monkeypatch.setattr("app.database.SessionLocal", factory) + import app.tasks.extraction as extraction_task + monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: _HUB_RESULT) + # resolve_entities isn't exercised on a from-scratch run (nothing existing + # to resolve against yet), so mocking summarize_entity alone is enough + # here and avoids also stubbing out the (unrelated) resolution LLM call + monkeypatch.setattr( + "app.services.llm_service.summarize_entity", + lambda *a, **k: {"summary": "first pass", "key_facts": [], "time_range": {"start": "unknown", "end": "unknown"}}, + ) + extraction_task.run_extraction(task_id_1) + + task_id_2 = str(uuid.uuid4()) + session.execute(text( + "INSERT INTO extraction_tasks (id, ontology_id, prompt_id, model_id, status, parameters, progress, error, created_at, updated_at) " + "VALUES (:id, 'o-llm', 'p-1', 'm-1', 'queued', :params, '{}'::json, NULL, now(), now())" + ), {"id": task_id_2, "params": '{"model_name": "mock-extractor", "constraints": []}'}) + session.commit() + + monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: _HUB_RESULT) + # resolve_entities still runs (4 new entities of the same type) and needs + # a harmless response so it doesn't attempt a real network call; the + # assertion this test cares about is that summarize_entity is skipped + monkeypatch.setattr("app.services.llm_service._call_llm", lambda *a, **k: '{"clusters": []}') + monkeypatch.setattr( + "app.services.llm_service.summarize_entity", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("summarizer should not be called again")), + ) + extraction_task.run_extraction(task_id_2) + + try: + status = session.execute(text( + "SELECT status, error FROM extraction_tasks WHERE id = :id" + ), {"id": task_id_2}).mappings().one() + assert status["status"] == "completed", status["error"] + properties = session.execute(text( + "SELECT properties FROM entities WHERE ontology_id = 'o-llm' AND name_cn = '借款人'" + )).scalar_one() + properties = json.loads(properties) if isinstance(properties, str) else properties + # untouched from the first run — the summarizer was never invoked again + assert properties["summary"] == "first pass" + finally: + session.close() diff --git a/backend/tests/evals/test_document_extraction_validators.py b/backend/tests/evals/test_document_extraction_validators.py index 46be2409..4229f6c0 100644 --- a/backend/tests/evals/test_document_extraction_validators.py +++ b/backend/tests/evals/test_document_extraction_validators.py @@ -3,7 +3,9 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from evals.document_extraction.validators import keyword_recall, dedup_gate, instance_leakage_gate +from evals.document_extraction.validators import ( + keyword_recall, dedup_gate, instance_leakage_gate, rule_leakage_gate, precision_recall, +) SAMPLE_RESULT = { "entities": [ @@ -101,3 +103,101 @@ def test_instance_leakage_gate_clean_for_concept_names(): "relations": [], "logic_rules": [], "actions": [], "instances": [], } assert instance_leakage_gate(result) == [] + + +def test_instance_leakage_gate_flags_enum_value_entity_names(): + result = { + "entities": [ + {"name_cn": "信用分层", "description": "概念实体"}, + {"name_cn": "A层", "description": "leaked enum value"}, + {"name_cn": "M2+", "description": "leaked enum value"}, + {"name_cn": "D轮融资", "description": "leaked enum value"}, + ], + "relations": [], "logic_rules": [], "actions": [], "instances": [], + } + findings = instance_leakage_gate(result) + flagged_names = {f["name"] for f in findings} + assert flagged_names == {"A层", "M2+", "D轮融资"} + + +def test_rule_leakage_gate_flags_business_rule_entity_names(): + result = { + "entities": [ + {"name_cn": "借款人", "description": "概念实体"}, + {"name_cn": "促销定价规则", "description": "leaked business rule"}, + {"name_cn": "风险准入政策", "description": "leaked business rule"}, + ], + "relations": [], "logic_rules": [], "actions": [], "instances": [], + } + findings = rule_leakage_gate(result) + flagged_names = {f["name"] for f in findings} + assert flagged_names == {"促销定价规则", "风险准入政策"} + + +def test_rule_leakage_gate_clean_for_concept_names(): + result = { + "entities": [ + {"name_cn": "借款人", "description": "概念实体"}, + {"name_cn": "信用分层", "description": "概念实体"}, + ], + "relations": [], "logic_rules": [], "actions": [], "instances": [], + } + assert rule_leakage_gate(result) == [] + + +def test_precision_recall_perfect_match(): + gold = ["借款人", "信用分层"] + result = {"entities": [{"name_cn": "借款人"}, {"name_cn": "信用分层"}]} + pr = precision_recall(gold, result) + assert pr["precision"] == 1.0 + assert pr["recall"] == 1.0 + assert pr["f1"] == 1.0 + assert pr["false_positive"] == 0 + assert pr["false_negative"] == 0 + + +def test_precision_recall_counts_extra_extracted_entities_as_false_positives(): + gold = ["借款人"] + result = {"entities": [{"name_cn": "借款人"}, {"name_cn": "促销定价规则"}]} + pr = precision_recall(gold, result) + assert pr["true_positive"] == 1 + assert pr["false_positive"] == 1 + assert pr["recall"] == 1.0 + assert pr["precision"] == 0.5 + assert pr["unmatched_extracted_entities"] == ["促销定价规则"] + + +def test_precision_recall_counts_missing_gold_entities_as_false_negatives(): + gold = ["借款人", "授信额度"] + result = {"entities": [{"name_cn": "借款人"}]} + pr = precision_recall(gold, result) + assert pr["true_positive"] == 1 + assert pr["false_negative"] == 1 + assert pr["precision"] == 1.0 + assert pr["recall"] == 0.5 + assert pr["missed_gold_entities"] == ["授信额度"] + + +def test_precision_recall_fuzzy_matches_minor_surface_form_differences(): + gold = ["产品研发部"] + result = {"entities": [{"name_cn": "产品研发部门"}]} + pr = precision_recall(gold, result) + assert pr["true_positive"] == 1 + assert pr["missed_gold_entities"] == [] + + +def test_precision_recall_does_not_double_count_one_extracted_entity_for_two_gold_entities(): + """Each extracted entity can satisfy at most one gold entity — a single + lucky match must not inflate recall for an unrelated second gold entity.""" + gold = ["借款人", "还款人"] + result = {"entities": [{"name_cn": "借款人"}]} + pr = precision_recall(gold, result) + assert pr["true_positive"] == 1 + assert pr["false_negative"] == 1 + + +def test_precision_recall_empty_gold_is_perfect_precision_no_recall_penalty(): + pr = precision_recall([], {"entities": [{"name_cn": "任意实体"}]}) + assert pr["recall"] == 1.0 # nothing was required + assert pr["false_positive"] == 1 + assert pr["precision"] == 0.0 diff --git a/backend/tests/runtime/test_snapshot_freshness.py b/backend/tests/runtime/test_snapshot_freshness.py index a8240004..3f83e6db 100644 --- a/backend/tests/runtime/test_snapshot_freshness.py +++ b/backend/tests/runtime/test_snapshot_freshness.py @@ -40,14 +40,9 @@ from app.services.runtime.snapshots import get_snapshot, materialize_refresh_snapshot from app.services.v2.incremental.operations import get_last_successful_refresh_run -# Captured once at import so every case in this module reasons about the -# same instant: `test_fresh_snapshot_exposes_cursor_lag_and_lineage` needs -# an exact lag match against a `compute_snapshot_freshness(..., now=FIXED_NOW)` -# call, while `evaluate_fixture_runtime_freshness` exercises the real -# `RuntimeService` (which reads real wall-clock time internally) moments -# later in the same test process — negligible drift against this module's -# hour/day-scale lag buckets. -FIXED_NOW = datetime.now(timezone.utc) +def _now() -> datetime: + """Capture wall-clock time per fixture/test, not once at module import.""" + return datetime.now(timezone.utc) DOMAIN = "00000000-0000-0000-0000-0000000000f0" ONTOLOGY_ID = "ontology-freshness-001" @@ -167,17 +162,18 @@ def _seed_completed_run(db, *, dataset_version_id: str, pipeline_run_id: str, re db.commit() -def materialize_refresh_fixture(db, *, refresh_run_id: str): +def materialize_refresh_fixture(db, *, refresh_run_id: str, now: datetime | None = None): """Seed one governed dataset/pipeline pair plus the named `RefreshRun` and materialize the snapshot it backs. `refresh_run_id` selects the case shape: a plain success id gets a - fresh cursor/lag anchored to `FIXED_NOW`; "soft-stale"/"hard-stale" get + fresh cursor/lag anchored to the supplied/current test time; "soft-stale"/"hard-stale" get a cursor whose lag already exceeds the shared default policy's soft/hard thresholds; "unknown-cursor" gets a successful run with no recorded cursor at all. """ release = _ensure_release(db) + now = now or _now() dataset_version_id = f"dv-{refresh_run_id}" # "refresh-success-001" is this module's canonical fresh case, matching # the exact `pipeline_run_ids == ["pipeline-run-001"]` assertion below. @@ -198,7 +194,7 @@ def materialize_refresh_fixture(db, *, refresh_run_id: str): ) else: lag = _CASE_LAG_SECONDS[refresh_run_id] - observed_at = FIXED_NOW - timedelta(seconds=lag) + observed_at = now - timedelta(seconds=lag) run = RefreshRun( id=refresh_run_id, source_id="source-freshness-001", resource="orders", policy="batch", config_version=1, cursor_contract="watermark_primary_key", @@ -296,9 +292,10 @@ def get_action_plan(db, plan_id): def test_fresh_snapshot_exposes_cursor_lag_and_lineage(db): - snapshot = materialize_refresh_fixture(db, refresh_run_id="refresh-success-001") + now = _now() + snapshot = materialize_refresh_fixture(db, refresh_run_id="refresh-success-001", now=now) view = compute_snapshot_freshness( - snapshot, now=FIXED_NOW, policy=FreshnessPolicy(max_lag_seconds=300, stale_action="deny"), + snapshot, now=now, policy=FreshnessPolicy(max_lag_seconds=300, stale_action="deny"), ) assert view.state == "fresh" assert view.lag_seconds == 120 @@ -328,7 +325,7 @@ def test_unrelated_refresh_run_cannot_stamp_freshness_onto_unconnected_dataset_v idempotency_key="idem-refresh-unrelated-001", cursor_after_json={ "watermark": None, "primary_key": "999", "opaque_value": None, - "observed_at": FIXED_NOW.isoformat(), + "observed_at": _now().isoformat(), }, fencing_token=1, lag_seconds=5, pipeline_run_id="pr-unrelated-001", ) @@ -429,6 +426,7 @@ def test_investigate_succeeds_with_real_production_shaped_refresh_provenance(db, refresh_run_id = "refresh-provenance-shape-001" dataset_version_id = f"dv-{refresh_run_id}" pipeline_run_id = f"pr-{refresh_run_id}" + now = _now() # The exact dict shape production code builds — no source_type/locator/ # content_hash key anywhere (see polling.py:630-638, event_ingest.py:668-678). @@ -439,7 +437,7 @@ def test_investigate_succeeds_with_real_production_shaped_refresh_provenance(db, "config_version": 1, "cursor_contract": "watermark_primary_key", "cursor_outcome": "advanced", - "source_observed_at": (FIXED_NOW - timedelta(seconds=60)).isoformat(), + "source_observed_at": (now - timedelta(seconds=60)).isoformat(), } pipeline = Pipeline(id=f"pipeline-{pipeline_run_id}", name=pipeline_run_id, spec={}) @@ -468,7 +466,7 @@ def test_investigate_succeeds_with_real_production_shaped_refresh_provenance(db, idempotency_key=f"idem-{refresh_run_id}", cursor_after_json={ "watermark": None, "primary_key": "100", "opaque_value": None, - "observed_at": (FIXED_NOW - timedelta(seconds=60)).isoformat(), + "observed_at": (now - timedelta(seconds=60)).isoformat(), }, fencing_token=1, lag_seconds=60, pipeline_run_id=pipeline_run_id, source_provenance=dict(provenance), diff --git a/backend/tests/test_extraction_chunking.py b/backend/tests/test_extraction_chunking.py new file mode 100644 index 00000000..1636aeeb --- /dev/null +++ b/backend/tests/test_extraction_chunking.py @@ -0,0 +1,50 @@ +"""Long-document chunking (graph-engineering playbook, scaling guidance): +a document over the size threshold is chunked at section boundaries with a +trailing overlap, so a single extraction call never has to hold the whole +document — short documents (the common case) pass through untouched. +""" +from app.tasks.extraction import _chunk_document, _split_sections + + +def test_short_document_is_not_chunked(): + md = "# 标题\n\n短文档内容。" + assert _chunk_document(md, max_chars=1000) == [md] + + +def test_long_document_splits_at_section_headers(): + md = ( + "# 第一节\n\n" + "甲" * 40 + "\n\n" + "# 第二节\n\n" + "乙" * 40 + "\n\n" + "# 第三节\n\n" + "丙" * 40 + ) + chunks = _chunk_document(md, max_chars=60, overlap_chars=0) + assert len(chunks) > 1 + assert "甲" * 40 in "".join(chunks) + assert "乙" * 40 in "".join(chunks) + assert "丙" * 40 in "".join(chunks) + # each section stays whole within its chunk, never split mid-section + for section in ("# 第一节", "# 第二节", "# 第三节"): + assert sum(section in c for c in chunks) == 1 + + +def test_chunk_boundary_repeats_overlap_from_previous_chunk(): + md = ( + "# 第一节\n\n" + "甲" * 40 + "\n\n" + "# 第二节\n\n" + "乙" * 40 + ) + chunks = _chunk_document(md, max_chars=50, overlap_chars=10) + assert len(chunks) == 2 + # the tail of chunk 1 reappears at the start of chunk 2 + assert chunks[0][-10:] in chunks[1] + + +def test_document_without_headers_falls_back_to_paragraph_splitting(): + md = "\n\n".join(f"段落{i}" * 20 for i in range(5)) + chunks = _chunk_document(md, max_chars=60, overlap_chars=0) + assert len(chunks) > 1 + for i in range(5): + assert sum(f"段落{i}" * 20 in c for c in chunks) == 1 + + +def test_split_sections_single_section_when_no_headers_and_short(): + assert _split_sections("一段没有标题的短文本") == ["一段没有标题的短文本"] diff --git a/backend/tests/test_extraction_model_selection.py b/backend/tests/test_extraction_model_selection.py new file mode 100644 index 00000000..efaf68fc --- /dev/null +++ b/backend/tests/test_extraction_model_selection.py @@ -0,0 +1,35 @@ +"""Tiered model selection (graph-engineering playbook, model selection): +resolution/relation-inference weigh conflicting evidence and should use a +stronger sibling model when the user picked a fast/cheap variant for +extraction — with zero new user-facing setting, reading only what's already +configured in ModelConfig.models. +""" +from app.tasks.extraction import _select_reasoning_model + + +def test_upgrades_from_fast_variant_to_stronger_sibling(): + assert _select_reasoning_model( + "deepseek-flash", ["deepseek-v4-pro", "deepseek-flash"] + ) == "deepseek-v4-pro" + + +def test_recognizes_common_fast_naming_conventions(): + assert _select_reasoning_model("gpt-4o-mini", ["gpt-4o", "gpt-4o-mini"]) == "gpt-4o" + assert _select_reasoning_model("claude-haiku", ["claude-sonnet", "claude-haiku"]) == "claude-sonnet" + + +def test_falls_back_to_same_model_when_not_a_fast_variant(): + """The user already picked the strong model — never downgrade it.""" + assert _select_reasoning_model("deepseek-v4-pro", ["deepseek-v4-pro", "deepseek-flash"]) == "deepseek-v4-pro" + + +def test_falls_back_to_same_model_when_no_stronger_sibling_configured(): + assert _select_reasoning_model("deepseek-flash", ["deepseek-flash"]) == "deepseek-flash" + assert _select_reasoning_model("deepseek-flash", []) == "deepseek-flash" + assert _select_reasoning_model("deepseek-flash", None) == "deepseek-flash" + + +def test_does_not_pick_another_fast_variant_as_the_upgrade(): + assert _select_reasoning_model( + "gpt-4o-mini", ["gpt-4o-mini", "claude-haiku"] + ) == "gpt-4o-mini" diff --git a/backend/tests/test_graph_diagnostics.py b/backend/tests/test_graph_diagnostics.py new file mode 100644 index 00000000..6d72185b --- /dev/null +++ b/backend/tests/test_graph_diagnostics.py @@ -0,0 +1,61 @@ +"""Graph connectivity diagnostics (graph-engineering playbook, Section V.E): +connected components, edge/node density, and hub nodes as a health signal +for whether entity resolution actually merged what it should have. +""" +from app.services.graph_diagnostics import compute_graph_diagnostics + + +def _entities(*names): + return [{"id": n, "name_cn": n} for n in names] + + +def test_single_connected_component_when_fully_linked(): + entities = _entities("借款人", "贷款", "还款记录") + edges = [("借款人", "贷款"), ("贷款", "还款记录")] + diag = compute_graph_diagnostics(entities, edges) + assert diag["connected_components"] == 1 + assert diag["largest_component_size"] == 3 + assert diag["isolated_entity_count"] == 0 + + +def test_fragmented_islands_are_counted_separately(): + entities = _entities("A", "B", "C", "D") + edges = [("A", "B"), ("C", "D")] + diag = compute_graph_diagnostics(entities, edges) + assert diag["connected_components"] == 2 + assert diag["largest_component_size"] == 2 + + +def test_isolated_entities_are_zero_degree_and_reported(): + entities = _entities("A", "B", "孤立实体") + edges = [("A", "B")] + diag = compute_graph_diagnostics(entities, edges) + assert diag["isolated_entity_count"] == 1 + assert diag["isolated_entities"] == ["孤立实体"] + + +def test_edge_node_ratio_and_hub_ranking(): + entities = _entities("hub", "a", "b", "c") + edges = [("hub", "a"), ("hub", "b"), ("hub", "c")] + diag = compute_graph_diagnostics(entities, edges) + assert diag["entity_count"] == 4 + assert diag["edge_count"] == 3 + assert diag["edge_node_ratio"] == 0.75 + assert diag["hub_entities"][0] == {"name_cn": "hub", "degree": 3} + + +def test_dangling_edge_endpoints_are_ignored(): + """An edge referencing an entity id not in the ontology (a data quality + issue caught elsewhere by P0 validation) must not crash diagnostics.""" + entities = _entities("A", "B") + edges = [("A", "B"), ("A", "不存在的实体")] + diag = compute_graph_diagnostics(entities, edges) + assert diag["edge_count"] == 1 + assert diag["connected_components"] == 1 + + +def test_empty_ontology_does_not_divide_by_zero(): + diag = compute_graph_diagnostics([], []) + assert diag["entity_count"] == 0 + assert diag["edge_node_ratio"] == 0.0 + assert diag["connected_components"] == 0 diff --git a/backend/tests/test_graph_files_regressions.py b/backend/tests/test_graph_files_regressions.py new file mode 100644 index 00000000..1c093905 --- /dev/null +++ b/backend/tests/test_graph_files_regressions.py @@ -0,0 +1,95 @@ +import io +import os + +import pytest + +from app.config import settings + + +def test_create_relation_rejects_missing_entity_key_without_500(client, auth_headers, ontology): + response = client.post( + f"/api/v1/ontologies/{ontology['id']}/graph/relations", + json={"source_entity": "missing-target"}, + headers=auth_headers, + ) + + assert response.status_code == 422 + + +def test_create_relation_rejects_entity_from_another_ontology(client, auth_headers, ontology): + first = client.post( + f"/api/v1/ontologies/{ontology['id']}/entities", + json={"name_cn": "First"}, + headers=auth_headers, + ).json()["data"] + other_ontology = client.post( + "/api/v1/ontologies", + json={"name": "Other ontology", "domain": "供应链"}, + headers=auth_headers, + ).json()["data"] + second = client.post( + f"/api/v1/ontologies/{other_ontology['id']}/entities", + json={"name_cn": "Second"}, + headers=auth_headers, + ).json()["data"] + + response = client.post( + f"/api/v1/ontologies/{ontology['id']}/graph/relations", + json={"source_entity": first["id"], "target_entity": second["id"]}, + headers=auth_headers, + ) + + assert response.status_code == 422 + + +def test_upload_conversion_failure_removes_partial_file(client, auth_headers, ontology, tmp_path, monkeypatch): + monkeypatch.setattr(settings, "uploads_dir", str(tmp_path)) + + class FailedConversion: + ok = False + content = None + error = "conversion failed" + + monkeypatch.setattr("app.routers.files.convert_document", lambda *_args: FailedConversion()) + response = client.post( + f"/api/v1/ontologies/{ontology['id']}/files", + files={"file": ("bad.txt", io.BytesIO(b"not usable"), "text/plain")}, + headers=auth_headers, + ) + + assert response.status_code == 422 + assert list(tmp_path.rglob("*")) == [tmp_path / ontology["id"]] + + +def test_delete_file_does_not_remove_disk_file_before_commit( + client, auth_headers, ontology, admin_user, db, tmp_path, monkeypatch, +): + monkeypatch.setattr(settings, "uploads_dir", str(tmp_path)) + uploaded = client.post( + f"/api/v1/ontologies/{ontology['id']}/files", + files={"file": ("ok.txt", io.BytesIO(b"hello"), "text/plain")}, + headers=auth_headers, + ) + assert uploaded.status_code == 201 + # The public response intentionally omits file_path; discover it from the + # temporary ontology directory instead. + paths = [path for path in (tmp_path / ontology["id"]).iterdir() if path.is_file()] + assert len(paths) == 1 + file_path = paths[0] + + from app.routers import files as files_router + + original_commit = type(db).commit + + def failing_commit(self): + raise RuntimeError("commit failed") + + monkeypatch.setattr(type(db), "commit", failing_commit) + try: + with pytest.raises(RuntimeError): + files_router.delete_file( + ontology["id"], uploaded.json()["data"]["id"], db, current_user=admin_user, + ) + finally: + monkeypatch.setattr(type(db), "commit", original_commit) + assert os.path.exists(file_path) diff --git a/backend/tests/test_ontologies.py b/backend/tests/test_ontologies.py index 754a9506..03680d76 100644 --- a/backend/tests/test_ontologies.py +++ b/backend/tests/test_ontologies.py @@ -1,3 +1,51 @@ +def test_delete_ontology_cleans_up_neo4j_and_chroma(client, auth_headers, monkeypatch): + """Deleting an ontology must also remove its Neo4j nodes/relations and its + Chroma vector collection — those stores have no other owner once the + Postgres row is gone, and previously nothing in the delete path ever + called their (already-implemented) delete methods.""" + calls = {} + + class FakeNeo4j: + def delete_by_ontology(self, ontology_id): + calls["neo4j_ontology_id"] = ontology_id + return 3 + + class FakeChroma: + def delete_collection(self, name): + calls["chroma_collection"] = name + return True + + monkeypatch.setattr("app.services.v2.graph.neo4j_service.Neo4jService", FakeNeo4j) + monkeypatch.setattr("app.services.v2.vector.chroma_service.ChromaService", FakeChroma) + + r = client.post("/api/v1/ontologies", json={"name": "CleanupTest", "domain": "财务"}, headers=auth_headers) + oid = r.json()["data"]["id"] + r2 = client.delete(f"/api/v1/ontologies/{oid}", headers=auth_headers) + assert r2.status_code == 204 + assert calls["neo4j_ontology_id"] == oid + assert calls["chroma_collection"] == f"ontology_{oid}" + + +def test_delete_ontology_survives_neo4j_and_chroma_failure(client, auth_headers, monkeypatch): + """External-store cleanup is best-effort: a Postgres delete that already + committed must still return 204 even if Neo4j/Chroma cleanup blows up.""" + class BrokenNeo4j: + def delete_by_ontology(self, ontology_id): + raise ConnectionError("neo4j unreachable") + + class BrokenChroma: + def delete_collection(self, name): + raise ConnectionError("chroma unreachable") + + monkeypatch.setattr("app.services.v2.graph.neo4j_service.Neo4jService", BrokenNeo4j) + monkeypatch.setattr("app.services.v2.vector.chroma_service.ChromaService", BrokenChroma) + + r = client.post("/api/v1/ontologies", json={"name": "CleanupFailTest", "domain": "财务"}, headers=auth_headers) + oid = r.json()["data"]["id"] + r2 = client.delete(f"/api/v1/ontologies/{oid}", headers=auth_headers) + assert r2.status_code == 204 + + def test_create_ontology(client, auth_headers): r = client.post("/api/v1/ontologies", json={"name": "供应链测试", "domain": "供应链"}, @@ -31,6 +79,15 @@ def test_get_ontology(client, auth_headers): assert r2.status_code == 200 assert r2.json()["data"]["name"] == "GetTest" +def test_get_ontology_exposes_is_dirty(client, auth_headers, db): + from app.models.ontology import OntologyProject + r = client.post("/api/v1/ontologies", json={"name": "DirtyTest", "domain": "财务"}, headers=auth_headers) + oid = r.json()["data"]["id"] + assert client.get(f"/api/v1/ontologies/{oid}", headers=auth_headers).json()["data"]["is_dirty"] is False + db.query(OntologyProject).filter(OntologyProject.id == oid).update({"is_dirty": True}) + db.commit() + assert client.get(f"/api/v1/ontologies/{oid}", headers=auth_headers).json()["data"]["is_dirty"] is True + def test_delete_ontology(client, auth_headers): r = client.post("/api/v1/ontologies", json={"name": "Del", "domain": "财务"}, headers=auth_headers) oid = r.json()["data"]["id"] diff --git a/backend/tests/test_ontology_deletion_with_instances.py b/backend/tests/test_ontology_deletion_with_instances.py new file mode 100644 index 00000000..ecc75074 --- /dev/null +++ b/backend/tests/test_ontology_deletion_with_instances.py @@ -0,0 +1,107 @@ +"""Deleting an ontology that has entity_instance_relations data must not hit +ONTOLOGY_HAS_DEPENDENT_DATA — the AFTER DELETE trigger on entity_instance_relations +(and entity_instances) re-emits agent_index_outbox rows, which must be swept up +by cleaning agent_index_outbox LAST, not before those tables. Needs a real +Postgres schema (SQLite has none of these triggers).""" +import os +import uuid +from urllib.parse import quote + +import pytest +from sqlalchemy import create_engine, text +from sqlalchemy.orm import sessionmaker + +from app.routers.ontologies import delete_ontology + +TEST_DATABASE_URL = os.environ.get("TEST_DATABASE_URL") +DEFAULT_DOMAIN = "00000000-0000-0000-0000-000000000001" + + +def _scoped_url(schema: str) -> str: + return f"{TEST_DATABASE_URL}?options={quote(f'-csearch_path={schema},public', safe='-=,')}" + + +@pytest.fixture +def schema(): + if not TEST_DATABASE_URL: + pytest.skip("TEST_DATABASE_URL required") + schema = "ontology_delete_" + uuid.uuid4().hex + engine = create_engine(TEST_DATABASE_URL) + with engine.begin() as connection: + connection.execute(text(f'CREATE SCHEMA "{schema}"')) + import subprocess + import sys + from pathlib import Path + backend_dir = Path(__file__).resolve().parents[1] + result = subprocess.run( + [sys.executable, "scripts/run_migrations.py", "upgrade", "head"], + cwd=backend_dir, env=dict(os.environ, DATABASE_URL=_scoped_url(schema)), + capture_output=True, text=True, + ) + assert result.returncode == 0, result.stderr + yield schema + with engine.begin() as connection: + connection.execute(text(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE')) + engine.dispose() + + +def test_delete_ontology_with_instance_relations_does_not_hit_dependent_data(schema, monkeypatch): + engine = create_engine(_scoped_url(schema)) + session = sessionmaker(bind=engine)() + try: + _run_delete_scenario(session) + finally: + # an open transaction would block the fixture's DROP SCHEMA forever + session.close() + engine.dispose() + + +def _run_delete_scenario(session): + from app.models.user import User + from app.services.ontology_access import backfill_creator_grants + + oid, u = "o-del-1", "u-del-1" + session.execute(text( + "INSERT INTO users (id,username,email,password_hash,role,is_active,security_domain_id,created_at,updated_at) " + "VALUES (:u,'d','d@t.com','h','admin',true,:d,now(),now())" + ), {"u": u, "d": DEFAULT_DOMAIN}) + session.execute(text( + "INSERT INTO ontology_projects (id,name,domain,version,status,created_by,created_at,updated_at,security_domain_id,working_revision) " + "VALUES (:o,'del-test','credit','v1','created',:u,now(),now(),:d,1)" + ), {"o": oid, "u": u, "d": DEFAULT_DOMAIN}) + session.execute(text( + "INSERT INTO entities (id, ontology_id, name_cn, type, description, properties, confidence, version, created_at, updated_at) " + "VALUES ('e-1', :o, '概念', 'Concept', 'd', '{}'::json, 0.9, 'v1', now(), now())" + ), {"o": oid}) + session.execute(text( + "INSERT INTO entity_instances (id, entity_id, ontology_id, row_identity, row_data, revision, created_at, updated_at) " + "VALUES (:i1, 'e-1', :o, 'inst-1', '{}'::json, 1, now(), now()), " + "(:i2, 'e-1', :o, 'inst-2', '{}'::json, 1, now(), now())" + ), {"o": oid, "i1": str(uuid.uuid4()), "i2": str(uuid.uuid4())}) + inst_ids = session.execute(text( + "SELECT id FROM entity_instances WHERE ontology_id = :o ORDER BY row_identity" + ), {"o": oid}).scalars().all() + session.execute(text( + "INSERT INTO relations (id, ontology_id, source_entity, target_entity, type, properties, confidence, created_at) " + "VALUES ('r-1', :o, 'e-1', 'e-1', 'related_to', '{}'::json, 0.9, now())" + ), {"o": oid}) + session.execute(text( + "INSERT INTO entity_instance_relations (id, ontology_id, source_instance_id, target_instance_id, relation_definition_id, properties, revision, created_at, updated_at) " + "VALUES (:id, :o, :s, :t, 'r-1', '{}'::json, 1, now(), now())" + ), {"id": str(uuid.uuid4()), "o": oid, "s": inst_ids[0], "t": inst_ids[1]}) + session.commit() + + # confirm the trigger fired on insert too (sanity: outbox has rows before delete) + outbox_before = session.execute(text( + "SELECT count(*) FROM agent_index_outbox WHERE ontology_id = :o" + ), {"o": oid}).scalar_one() + assert outbox_before > 0 + + # rows were inserted via direct SQL, bypassing create_ontology's creator grant + backfill_creator_grants(session) + delete_ontology(ontology_id=oid, db=session, current_user=session.get(User, u)) + + remaining = session.execute(text( + "SELECT count(*) FROM ontology_projects WHERE id = :o" + ), {"o": oid}).scalar_one() + assert remaining == 0 diff --git a/backend/tests/test_python_version.py b/backend/tests/test_python_version.py new file mode 100644 index 00000000..a0fdd391 --- /dev/null +++ b/backend/tests/test_python_version.py @@ -0,0 +1,18 @@ +import pytest + +from scripts import check_python_version + + +@pytest.mark.parametrize("version", [(3, 11, 0), (3, 12, 0)]) +def test_supported_python_versions_are_accepted(monkeypatch, version): + monkeypatch.setattr(check_python_version.sys, "version_info", version) + + assert check_python_version.require_supported_python() is True + + +@pytest.mark.parametrize("version", [(3, 10, 0), (3, 13, 0)]) +def test_python_versions_outside_supported_range_are_rejected(monkeypatch, version): + monkeypatch.setattr(check_python_version.sys, "version_info", version) + + with pytest.raises(SystemExit, match="UNSUPPORTED_PYTHON_VERSION"): + check_python_version.require_supported_python() diff --git a/backend/tests/v2/connection/test_connection_resources.py b/backend/tests/v2/connection/test_connection_resources.py new file mode 100644 index 00000000..5d7ce171 --- /dev/null +++ b/backend/tests/v2/connection/test_connection_resources.py @@ -0,0 +1,138 @@ +"""v2 saved database connection resource and sample API coverage.""" +import json +from unittest.mock import MagicMock, patch + +from app.models.v2.connection import Connection +from app.models.user import User +from app.routers.v2 import connections as connections_router +from app.services.auth_service import create_access_token, hash_password + + +def _override_connections_db(db): + def override(): + yield db + + return override + + +def test_saved_connection_keeps_credentials_encrypted_and_lists_resources( + client, db, auth_headers, admin_user, monkeypatch, +): + app = client.app + previous = app.dependency_overrides.get(connections_router.get_db) + app.dependency_overrides[connections_router.get_db] = _override_connections_db(db) + try: + created = client.post( + "/api/v2/connections", + headers=auth_headers, + json={ + "name": "runtime mysql", + "kind": "mysql", + "config": { + "host": "127.0.0.1", + "port": "53306", + "database": "runtime", + "user": "runtime", + "password": "secret", + }, + }, + ) + assert created.status_code == 201 + connection = db.get(Connection, created.json()["id"]) + assert connection is not None + assert "secret" not in json.dumps(connection.config) + assert "_encrypted" in connection.config + assert connection.created_by == admin_user.id + + fake = MagicMock() + fake.list_resources.return_value = ["codex_pipeline_probe"] + monkeypatch.setattr(connections_router, "get_connector", lambda kind, config: fake) + listed = client.get( + f"/api/v2/connections/{connection.id}/resources", + headers=auth_headers, + ) + + assert listed.status_code == 200 + assert listed.json()["resources"] == ["codex_pipeline_probe"] + fake.list_resources.assert_called_once_with() + finally: + if previous is None: + app.dependency_overrides.pop(connections_router.get_db, None) + else: + app.dependency_overrides[connections_router.get_db] = previous + + +def test_saved_connection_returns_sample_rows_for_a_table(client, db, auth_headers, monkeypatch): + app = client.app + previous = app.dependency_overrides.get(connections_router.get_db) + app.dependency_overrides[connections_router.get_db] = _override_connections_db(db) + try: + connection = Connection( + id="resource-sample-connection", + name="runtime postgres", + kind="postgres", + config={"_encrypted": "ignored-by-mock"}, + status="inactive", + ) + db.add(connection) + db.commit() + + fake = MagicMock() + fake.pull_sample.return_value = [{"id": 1, "customer": "Alice"}] + monkeypatch.setattr(connections_router, "_load_connection_connector", lambda *args, **kwargs: fake) + response = client.get( + f"/api/v2/connections/{connection.id}/resources/codex_pipeline_probe/sample?limit=1", + headers=auth_headers, + ) + + assert response.status_code == 200 + assert response.json()["rows"] == [{"id": 1, "customer": "Alice"}] + fake.pull_sample.assert_called_once_with("codex_pipeline_probe", limit=1) + finally: + if previous is None: + app.dependency_overrides.pop(connections_router.get_db, None) + else: + app.dependency_overrides[connections_router.get_db] = previous + + +def test_viewer_cannot_list_or_sample_saved_connection_resources(client, db): + app = client.app + previous = app.dependency_overrides.get(connections_router.get_db) + app.dependency_overrides[connections_router.get_db] = _override_connections_db(db) + try: + db.add(Connection( + id="viewer-resource-connection", + name="runtime postgres", + kind="postgres", + config={"_encrypted": "ignored-by-authz-test"}, + status="active", + )) + viewer = User( + id="resource-viewer", + username="resource-viewer", + email="resource-viewer@test.com", + password_hash=hash_password("viewer123"), + role="viewer", + ) + db.add(viewer) + db.commit() + headers = { + "Authorization": f"Bearer {create_access_token({'sub': viewer.id, 'role': viewer.role})}", + } + + listed = client.get( + "/api/v2/connections/viewer-resource-connection/resources", + headers=headers, + ) + sampled = client.get( + "/api/v2/connections/viewer-resource-connection/resources/orders/sample", + headers=headers, + ) + + assert listed.status_code == 403 + assert sampled.status_code == 403 + finally: + if previous is None: + app.dependency_overrides.pop(connections_router.get_db, None) + else: + app.dependency_overrides[connections_router.get_db] = previous diff --git a/backend/tests/v2/connection/test_scheduler.py b/backend/tests/v2/connection/test_scheduler.py index 1c086e89..287753e3 100644 --- a/backend/tests/v2/connection/test_scheduler.py +++ b/backend/tests/v2/connection/test_scheduler.py @@ -47,9 +47,8 @@ def test_schedule_connection_without_db_only_validates_never_reports_active(): def test_schedule_pipeline_without_db_only_validates_never_reports_active(): svc = CronService() - result = svc.schedule_pipeline_run("pl-1", "*/30 * * * *") - assert result["status"] != "scheduled" - assert result["pipeline_id"] == "pl-1" + with pytest.raises(ValueError, match="PIPELINE_SCHEDULE_UNSUPPORTED"): + svc.schedule_pipeline_run("pl-1", "*/30 * * * *") def test_schedule_connection_with_db_persists_and_returns_scheduled(db): @@ -69,10 +68,13 @@ def test_schedule_connection_with_db_persists_and_returns_scheduled(db): def test_schedule_pipeline_with_db_persists_and_returns_scheduled(db): svc = CronService() - result = svc.schedule_pipeline_run("pl-1", "*/30 * * * *", db=db) - assert result["status"] == "scheduled" - assert result["pipeline_id"] == "pl-1" - assert result["next_due_at"] is not None + with pytest.raises(ValueError, match="PIPELINE_SCHEDULE_UNSUPPORTED"): + svc.schedule_pipeline_run("pl-1", "*/30 * * * *", db=db) + + from app.models.v2.refresh import RefreshSchedule + assert db.query(RefreshSchedule).filter( + RefreshSchedule.target_type == "pipeline", RefreshSchedule.target_id == "pl-1", + ).count() == 0 def test_describe_cron_every_minute(): @@ -137,11 +139,11 @@ def scheduler_connection(db): db.add(Connection( id="scheduler-conn-001", name="scheduler-source", kind="rest", status="active", - config={"cursor_contract": "watermark_primary_key"}, + config={"cursor_contract": "watermark_primary_key", "resource": "orders"}, refresh_policy="micro_batch", cursor_contract="watermark_primary_key", )) db.add(RefreshSourceState( - id="scheduler-state-001", source_id="scheduler-conn-001", resource="__scheduled__", + id="scheduler-state-001", source_id="scheduler-conn-001", resource="orders", cursor_contract="watermark_primary_key", config_version=1, cursor_json={ "watermark": NOW.isoformat(), "primary_key": "1", "opaque_value": None, diff --git a/backend/tests/v2/connection/test_sql_connector.py b/backend/tests/v2/connection/test_sql_connector.py index 46acd331..79d34553 100644 --- a/backend/tests/v2/connection/test_sql_connector.py +++ b/backend/tests/v2/connection/test_sql_connector.py @@ -37,6 +37,49 @@ def test_registry_mysql_returns_sql_connector(): assert CONNECTOR_REGISTRY["postgres"] is SQLConnector +def test_saved_sql_connection_fields_are_compiled_to_a_safe_sqlalchemy_url(): + from app.services.connection.config import build_connector_config + + mysql = build_connector_config( + "mysql", + { + "host": "127.0.0.1", + "port": "3306", + "database": "runtime", + "user": "runtime", + "password": "p@ss:word/with#chars", + }, + ) + postgres = build_connector_config( + "postgres", + { + "host": "127.0.0.1", + "port": 5432, + "database": "runtime", + "user": "runtime", + "password": "secret", + }, + ) + + assert mysql["connection_string"] == ( + "mysql+pymysql://runtime:p%40ss%3Aword%2Fwith%23chars@127.0.0.1:3306/runtime" + ) + assert postgres["connection_string"] == "postgresql+psycopg2://runtime:secret@127.0.0.1:5432/runtime" + + +def test_sql_connector_keeps_actionable_failure_detail_without_exposing_password(): + from app.services.connection.sql_connector import SQLConnector + + connector = SQLConnector({ + "connection_string": "postgresql+psycopg2://user:secret@localhost:5432/runtime", + }) + connector._get_engine = MagicMock(side_effect=RuntimeError("connection refused")) + + assert connector.test_connection() is False + assert connector.last_error == "connection refused" + assert "secret" not in connector.last_error + + def test_get_connector_all_supported_kinds(): from app.services.connection.registry import get_connector for kind in ["file", "mysql", "postgres"]: diff --git a/backend/tests/v2/graph/test_analytics.py b/backend/tests/v2/graph/test_analytics.py index 34e83d0b..dc79b532 100644 --- a/backend/tests/v2/graph/test_analytics.py +++ b/backend/tests/v2/graph/test_analytics.py @@ -63,3 +63,39 @@ def test_node_degree_with_result(): result = svc.node_degree("ont-1", "node-1") assert result["in_degree"] == 3 assert result["out_degree"] == 7 + + +def test_graph_traversals_scope_every_returned_node_to_ontology(): + svc, mock_neo4j = make_analytics(available=True) + mock_neo4j.run_cypher.return_value = [] + + svc.get_neighbors("ont-1", "node-1") + neighbors_query = mock_neo4j.run_cypher.call_args.args[0] + assert "all(node IN nodes(path) WHERE node.ontology_id = $ontology_id)" in neighbors_query + + svc.shortest_path("ont-1", "a", "b") + path_query = mock_neo4j.run_cypher.call_args.args[0] + assert "all(node IN nodes(p) WHERE node.ontology_id = $ontology_id)" in path_query + + svc.node_degree("ont-1", "node-1") + degree_query = mock_neo4j.run_cypher.call_args.args[0] + assert "out_neighbor.ontology_id = $ontology_id" in degree_query + assert "in_neighbor.ontology_id = $ontology_id" in degree_query + + svc.top_connected_nodes("ont-1") + top_query = mock_neo4j.run_cypher.call_args.args[0] + assert "neighbor.ontology_id = $ontology_id" in top_query + + +def test_graph_neighbors_route_scopes_every_path_node(monkeypatch): + from app.routers.v2 import graph + + fake_neo4j = MagicMock() + fake_neo4j.available = True + fake_neo4j.run_cypher.return_value = [] + monkeypatch.setattr(graph, "get_neo4j", lambda: fake_neo4j) + + graph.get_neighbors("ont-1", "node-1") + + query = fake_neo4j.run_cypher.call_args.args[0] + assert "all(node IN nodes(path) WHERE node.ontology_id = $ontology_id)" in query diff --git a/backend/tests/v2/incremental/test_refresh_schedule.py b/backend/tests/v2/incremental/test_refresh_schedule.py index 9781dd4b..7933ecc3 100644 --- a/backend/tests/v2/incremental/test_refresh_schedule.py +++ b/backend/tests/v2/incremental/test_refresh_schedule.py @@ -24,9 +24,9 @@ import pytest from sqlalchemy import func, select +from app.models.v2.connection import Connection from app.models.v2.refresh import RefreshRun, RefreshSchedule from app.services.v2.scheduler.schedule_service import ( - DEFAULT_RESOURCE, ScheduleRequest, dispatch_due_schedules, upsert_refresh_schedule, @@ -51,6 +51,12 @@ def upsert_fixture_schedule(db, *, cron_expr, timezone, business_calendar, sla_s def create_fixture_schedule(db, *, target_id, max_pending_runs=5, cron_expr="* * * * *", retry_policy=None) -> RefreshSchedule: """A schedule already due at FIXED_NOW (every-minute cron, upserted one minute before FIXED_NOW so next_due_at lands exactly on it).""" + if db.get(Connection, target_id) is None: + db.add(Connection( + id=target_id, name=target_id, kind="rest", config={"resource": "orders"}, + status="active", refresh_policy="batch", cursor_contract="watermark_primary_key", + )) + db.commit() request = ScheduleRequest( target_type="connection", target_id=target_id, cron_expr=cron_expr, timezone="UTC", business_calendar=[], sla_seconds=0, retry_policy=retry_policy, backfill_window_seconds=0, @@ -59,7 +65,7 @@ def create_fixture_schedule(db, *, target_id, max_pending_runs=5, cron_expr="* * return upsert_refresh_schedule(db, request, now=FIXED_NOW - timedelta(minutes=1)) -def create_queued_fixture_run(db, *, source_id, resource=DEFAULT_RESOURCE) -> RefreshRun: +def create_queued_fixture_run(db, *, source_id, resource="orders") -> RefreshRun: """A pre-existing active run occupying the source's admission slot, independent of any schedule dispatch.""" run = RefreshRun( @@ -123,6 +129,12 @@ def test_scheduled_run_persists_destination_queue_for_operability(db): def test_manual_refresh_run_persists_destination_queue_for_operability(db): from app.tasks.v2.refresh_tasks import create_manual_connection_run + db.add(Connection( + id="source-manual-queue-attribution", name="manual source", kind="rest", + config={"resource": "orders"}, status="active", refresh_policy="batch", + cursor_contract="watermark_primary_key", + )) + db.commit() run = create_manual_connection_run(db, "source-manual-queue-attribution") assert run.dispatch_queue == "refresh.poll" @@ -287,7 +299,7 @@ def test_expired_dispatch_claim_is_recovered_once_without_duplicate(db): schedule = create_fixture_schedule(db, target_id="source-expired-claim") occurrence_key = f"schedule:{schedule.id}:{schedule.next_due_at.isoformat()}" retained = RefreshRun( - id=str(uuid.uuid4()), source_id="source-expired-claim", resource=DEFAULT_RESOURCE, + id=str(uuid.uuid4()), source_id="source-expired-claim", resource="orders", policy="batch", trigger="scheduled", config_version=1, cursor_contract="watermark_primary_key", status="queued", dispatch_state="pending", dispatch_queue="refresh.poll", dispatch_claim_owner="abandoned-beat", diff --git a/backend/tests/v2/incremental/test_resource_resolution.py b/backend/tests/v2/incremental/test_resource_resolution.py new file mode 100644 index 00000000..a6c223f8 --- /dev/null +++ b/backend/tests/v2/incremental/test_resource_resolution.py @@ -0,0 +1,195 @@ +"""Deterministic server-owned resource selection for refresh entry points.""" +from datetime import datetime, timedelta, timezone + +import pytest + +from app.models.v2.connection import Connection +from app.models.v2.dataset import Dataset +from app.models.v2.refresh import RefreshRun, RefreshSchedule, RefreshSourceState +from app.schemas.refresh import RefreshError, RefreshPolicy +from app.services.v2.incremental.operations import resolve_refresh_resource +from app.services.v2.scheduler.schedule_service import dispatch_due_schedules, upsert_refresh_schedule, ScheduleRequest +from app.tasks.v2.refresh_tasks import create_manual_connection_run + + +NOW = datetime(2026, 9, 24, tzinfo=timezone.utc) + + +def _connection(db, connection_id="resource-source", config=None): + connection = Connection( + id=connection_id, + name=connection_id, + kind="rest", + config=config or {}, + status="active", + refresh_policy=RefreshPolicy.BATCH.value, + cursor_contract="watermark_primary_key", + ) + db.add(connection) + db.commit() + return connection + + +def test_resource_config_wins_over_ambiguous_states(db): + _connection(db, config={"resource": "orders"}) + db.add_all([ + RefreshSourceState( + id="state-users", source_id="resource-source", resource="users", + cursor_contract="watermark_primary_key", config_version=1, + fencing_token=0, + ), + RefreshSourceState( + id="state-events", source_id="resource-source", resource="events", + cursor_contract="watermark_primary_key", config_version=1, + fencing_token=0, + ), + ]) + db.commit() + + assert resolve_refresh_resource(db, source_id="resource-source") == "orders" + + +def test_resource_resolution_ignores_legacy_sentinel_and_uses_one_real_state(db): + _connection(db) + db.add(RefreshSourceState( + id="state-sentinel", source_id="resource-source", resource="__scheduled__", + cursor_contract="watermark_primary_key", config_version=1, fencing_token=0, + )) + db.commit() + + with pytest.raises(RefreshError) as exc_info: + resolve_refresh_resource(db, source_id="resource-source") + assert exc_info.value.reason_code == "REFRESH_RESOURCE_REQUIRED" + + db.add(RefreshSourceState( + id="state-orders", source_id="resource-source", resource="orders", + cursor_contract="watermark_primary_key", config_version=1, fencing_token=0, + )) + db.commit() + assert resolve_refresh_resource(db, source_id="resource-source") == "orders" + + +def test_resource_resolution_uses_one_dataset_when_state_is_absent(db): + _connection(db) + db.add(Dataset( + id="dataset-orders", name="orders", source_connection_id="resource-source", + kind="structured", + )) + db.commit() + + assert resolve_refresh_resource(db, source_id="resource-source") == "orders" + + +def test_resource_resolution_rejects_ambiguous_state_candidates(db): + _connection(db) + db.add_all([ + RefreshSourceState( + id="state-orders", source_id="resource-source", resource="orders", + cursor_contract="watermark_primary_key", config_version=1, fencing_token=0, + ), + RefreshSourceState( + id="state-users", source_id="resource-source", resource="users", + cursor_contract="watermark_primary_key", config_version=1, fencing_token=0, + ), + ]) + db.commit() + + with pytest.raises(RefreshError) as exc_info: + resolve_refresh_resource(db, source_id="resource-source") + assert exc_info.value.reason_code == "REFRESH_RESOURCE_AMBIGUOUS" + + +def test_manual_compatibility_run_uses_concrete_resolved_resource(db): + _connection(db, config={"resource": "orders"}) + + run = create_manual_connection_run(db, "resource-source") + + assert run.resource == "orders" + assert db.query(RefreshRun).filter( + RefreshRun.source_id == "resource-source", RefreshRun.resource == "__scheduled__", + ).count() == 0 + + +def test_due_source_schedule_uses_concrete_resolved_resource(db): + _connection(db, config={"resource": "orders"}) + schedule = upsert_refresh_schedule( + db, + ScheduleRequest( + target_type="connection", target_id="resource-source", cron_expr="* * * * *", + timezone="UTC", business_calendar=[], sla_seconds=0, retry_policy=None, + backfill_window_seconds=0, max_pending_runs=1, enabled=True, + ), + now=NOW - timedelta(minutes=1), + ) + + sent = [] + dispatched = dispatch_due_schedules( + db, now=NOW, lease_owner="beat", send_refresh=lambda message: sent.append(message) or message.run_id, + ) + + run = db.query(RefreshRun).filter(RefreshRun.idempotency_key.like(f"schedule:{schedule.id}:%")).one() + assert dispatched == [run.id] + assert run.resource == "orders" + assert run.resource != "__scheduled__" + assert sent[0].task_name == "refresh.connection" + + +def test_pipeline_target_schedule_is_rejected_before_persistence(db): + from app.services.v2.scheduler.cron_service import CronService + + with pytest.raises(ValueError, match="PIPELINE_SCHEDULE_UNSUPPORTED"): + CronService().schedule_pipeline_run("pipeline-1", "*/30 * * * *", db=db) + + assert db.query(RefreshSchedule).filter( + RefreshSchedule.target_type == "pipeline", RefreshSchedule.target_id == "pipeline-1", + ).count() == 0 + + +def test_existing_pipeline_schedule_is_disabled_without_poll_run(db): + schedule = RefreshSchedule( + id="pipeline-schedule-1", target_type="pipeline", target_id="pipeline-1", + cron_expression="* * * * *", timezone="UTC", excluded_dates=[], + max_pending_runs=1, enabled=True, next_due_at=NOW, + ) + db.add(schedule) + db.commit() + + assert dispatch_due_schedules( + db, now=NOW, lease_owner="beat", send_refresh=lambda message: message.run_id, + ) == [] + db.refresh(schedule) + assert schedule.enabled is False + assert db.query(RefreshRun).filter(RefreshRun.source_id == "pipeline-1").count() == 0 + + +def test_bad_resource_schedule_does_not_block_good_schedule(db): + _connection(db, connection_id="good-resource-source", config={"resource": "orders"}) + _connection(db, connection_id="bad-resource-source") + good = upsert_refresh_schedule( + db, + ScheduleRequest( + target_type="connection", target_id="good-resource-source", cron_expr="* * * * *", + timezone="UTC", business_calendar=[], sla_seconds=0, retry_policy=None, + backfill_window_seconds=0, max_pending_runs=1, enabled=True, + ), + now=NOW - timedelta(minutes=1), + ) + bad = upsert_refresh_schedule( + db, + ScheduleRequest( + target_type="connection", target_id="bad-resource-source", cron_expr="* * * * *", + timezone="UTC", business_calendar=[], sla_seconds=0, retry_policy=None, + backfill_window_seconds=0, max_pending_runs=1, enabled=True, + ), + now=NOW - timedelta(minutes=1), + ) + + dispatched = dispatch_due_schedules( + db, now=NOW, lease_owner="beat", send_refresh=lambda message: message.run_id, + ) + + assert len(dispatched) == 1 + run = db.query(RefreshRun).filter(RefreshRun.idempotency_key.like(f"schedule:{good.id}:%")).one() + assert dispatched == [run.id] + db.refresh(bad) + assert bad.next_due_at.replace(tzinfo=timezone.utc) > NOW diff --git a/backend/tests/v2/pipeline/test_database_source.py b/backend/tests/v2/pipeline/test_database_source.py new file mode 100644 index 00000000..7d5925ad --- /dev/null +++ b/backend/tests/v2/pipeline/test_database_source.py @@ -0,0 +1,126 @@ +"""Database connector source -> snapshot -> Route A pipeline coverage.""" +from unittest.mock import MagicMock + +from sqlalchemy.orm import sessionmaker + +from app.models.v2.connection import Connection +from app.models.v2.pipeline import Pipeline, PipelineRun, PipelineRunInput +from app.services import encryption_service +from app.tasks.v2 import pipeline_run as pipeline_module +from app.routers.v2.pipelines import _sanitize_definition + + +def test_pipeline_definition_strips_database_credentials_before_persisting(): + definition = { + "nodes": [{ + "type": "connector", + "config": { + "source_type": "postgresql", + "connection_id": "db-source-001", + "resource": "codex_pipeline_probe", + "config_values": { + "host": "127.0.0.1", + "password": "do-not-store", + }, + "password": "do-not-store-either", + "credentials": { + "username": "runtime", + "password": "nested-do-not-store", + }, + }, + }], + "edges": [], + } + + sanitized = _sanitize_definition(definition) + + assert sanitized is not definition + config = sanitized["nodes"][0]["config"] + assert config["connection_id"] == "db-source-001" + assert "password" not in config + assert "config_values" not in config + assert "credentials" not in config + assert definition["nodes"][0]["config"]["password"] == "do-not-store-either" + + +def test_pipeline_run_snapshots_selected_database_table_and_records_lineage(db, monkeypatch): + class MemoryStorage: + def __init__(self): + self.objects = {} + + def put_bytes(self, bucket, key, data, content_type="application/octet-stream"): + uri = f"s3://{bucket}/{key}" + self.objects[uri] = data + return uri + + def get_object(self, uri): + return self.objects[uri] + + storage = MemoryStorage() + monkeypatch.setattr("app.database.SessionLocal", sessionmaker(bind=db.get_bind())) + monkeypatch.setattr("app.services.v2.dataset_service.get_storage_service", lambda: storage) + connector = MagicMock() + connector.pull_full.return_value = [ + {"id": row_id, "customer": f"Customer {row_id}", "amount": row_id} + for row_id in range(10001) + ] + monkeypatch.setattr(pipeline_module, "get_connector", lambda kind, config: connector) + monkeypatch.setattr( + pipeline_module, + "_execute_route", + lambda route, context, data: (data, context), + ) + + connection = Connection( + id="db-source-001", + name="runtime postgres", + kind="postgres", + config={"_encrypted": encryption_service.encrypt( + '{"host":"127.0.0.1","port":"55432","database":"runtime",' + '"user":"runtime","password":"secret"}' + )}, + status="active", + ) + pipeline = Pipeline( + id="db-pipeline-001", + name="database pipeline", + route="A", + spec={}, + definition={ + "nodes": [{ + "id": "source", + "type": "connector", + "config": { + "source_type": "postgresql", + "connection_id": connection.id, + "resource": "codex_pipeline_probe", + }, + }], + "edges": [], + }, + status="draft", + ) + run = PipelineRun(id="db-run-001", pipeline_id=pipeline.id, status="pending") + db.add_all([connection, pipeline, run]) + db.commit() + + pipeline_module.pipeline_run_task.run(pipeline.id, run.id) + db.expire_all() + + persisted = db.get(PipelineRun, run.id) + assert persisted.status == "success" + assert persisted.dataset_version_id is not None + assert persisted.stats["rows_in"] == 10001 + assert db.query(PipelineRunInput).filter( + PipelineRunInput.pipeline_run_id == run.id, + ).count() >= 2 + connector.pull_full.assert_called_once_with("codex_pipeline_probe") + from app.models.v2.dataset import Dataset + from app.services.v2.incremental.polling import _resolve_pipeline + + source_dataset = db.query(Dataset).filter( + Dataset.source_connection_id == connection.id, + Dataset.name == "codex_pipeline_probe", + ).one() + assert db.get(Pipeline, pipeline.id).source_dataset_id == source_dataset.id + assert _resolve_pipeline(db, dataset=source_dataset).id == pipeline.id diff --git a/backend/tests/v2/pipeline/test_pinned_inputs.py b/backend/tests/v2/pipeline/test_pinned_inputs.py new file mode 100644 index 00000000..b24d9c47 --- /dev/null +++ b/backend/tests/v2/pipeline/test_pinned_inputs.py @@ -0,0 +1,104 @@ +"""Pipeline input pinning and pipeline-level status ordering regressions.""" +from datetime import datetime, timezone +from sqlalchemy.orm import sessionmaker + +from app.models.v2.dataset import Dataset, DatasetVersion +from app.models.v2.pipeline import Pipeline, PipelineRun +from app.tasks.v2 import pipeline_run as pipeline_module + + +def _file_pipeline(db, *, pipeline_id="pinned-pipeline"): + first = Dataset(id=f"{pipeline_id}-dataset-a", name="a", kind="structured") + second = Dataset(id=f"{pipeline_id}-dataset-b", name="b", kind="structured") + first_version = DatasetVersion( + id=f"{pipeline_id}-version-a", dataset_id=first.id, version_no=1, rowcount=1, + ) + second_version = DatasetVersion( + id=f"{pipeline_id}-version-b", dataset_id=second.id, version_no=1, rowcount=1, + ) + pipeline = Pipeline( + id=pipeline_id, name=pipeline_id, spec={}, status="draft", + definition={ + "nodes": [{ + "id": "source-a", "type": "connector", + "config": {"source_type": "file", "files": [{"dataset_id": first.id, "name": "a.csv"}]}, + }, { + "id": "source-b", "type": "connector", + "config": {"source_type": "file", "files": [{"dataset_id": second.id, "name": "b.csv"}]}, + }], + "edges": [], + }, + ) + db.add_all([first, second, first_version, second_version, pipeline]) + db.commit() + return pipeline, first_version, second_version + + +def test_incomplete_pinned_inputs_fail_before_latest_lookup(db, monkeypatch): + pipeline, first_version, _ = _file_pipeline(db) + run = PipelineRun(id="pinned-run-incomplete", pipeline_id=pipeline.id, status="pending") + db.add(run) + db.commit() + + def fail_if_latest_lookup(*args, **kwargs): + raise AssertionError("latest version lookup must not run for incomplete pins") + + monkeypatch.setattr( + "app.services.v2.incremental.polling.select_pinned_dataset_version", + fail_if_latest_lookup, + ) + monkeypatch.setattr("app.database.SessionLocal", sessionmaker(bind=db.get_bind())) + pipeline_module.pipeline_run_task.run(pipeline.id, run.id, [first_version.id]) + db.expire_all() + + persisted = db.get(PipelineRun, run.id) + assert persisted.status == "failed" + assert persisted.error_log.startswith("PINNED_INPUTS_INCOMPLETE") + + +def test_duplicate_pins_for_one_source_are_rejected(db, monkeypatch): + pipeline, first_version, _ = _file_pipeline(db, pipeline_id="pinned-pipeline-duplicate") + replacement = DatasetVersion( + id="pinned-pipeline-duplicate-version-a-2", + dataset_id=first_version.dataset_id, + version_no=2, + rowcount=1, + ) + db.add(replacement) + run = PipelineRun(id="pinned-run-duplicate", pipeline_id=pipeline.id, status="pending") + db.add(run) + db.commit() + + monkeypatch.setattr("app.database.SessionLocal", sessionmaker(bind=db.get_bind())) + pipeline_module.pipeline_run_task.run( + pipeline.id, run.id, [first_version.id, replacement.id], + ) + db.expire_all() + + persisted = db.get(PipelineRun, run.id) + assert persisted.status == "failed" + assert persisted.error_log.startswith("PINNED_INPUTS_INCOMPLETE") + + +def test_pipeline_status_summary_only_follows_latest_run(db): + from app.tasks.v2.pipeline_run import _set_pipeline_status_if_latest + + pipeline = Pipeline(id="status-pipeline", name="status-pipeline", spec={}, status="draft") + older = PipelineRun( + id="status-run-a", pipeline_id=pipeline.id, status="failed", + finished_at=datetime(2026, 9, 24, 0, 0, tzinfo=timezone.utc), + error_log="older failure", + created_at=datetime(2026, 9, 24, 0, 0, tzinfo=timezone.utc), + ) + newer = PipelineRun( + id="status-run-b", pipeline_id=pipeline.id, status="running", + created_at=datetime(2026, 9, 24, 0, 1, tzinfo=timezone.utc), + ) + db.add_all([pipeline, older, newer]) + db.commit() + + assert _set_pipeline_status_if_latest(db, pipeline, older, "published") is False + assert db.get(Pipeline, pipeline.id).status == "draft" + + assert _set_pipeline_status_if_latest(db, pipeline, newer, "published") is True + assert db.get(Pipeline, pipeline.id).status == "published" diff --git a/backend/tests/v2/test_curated_deletion.py b/backend/tests/v2/test_curated_deletion.py new file mode 100644 index 00000000..355fbe1a --- /dev/null +++ b/backend/tests/v2/test_curated_deletion.py @@ -0,0 +1,106 @@ +"""Curated dataset deletion — MinIO object cleanup and legacy-table cleanup""" +import uuid +from datetime import datetime, timezone + +import pytest + +from app.models.v2.curated import CuratedDataset, CuratedReview +from app.models.v2.dataset import Dataset, DatasetVersion, MediaItem + + +@pytest.fixture +def curated_client(client, db): + """v2 curated 路由用模块内 get_db, 需单独覆盖到测试库""" + from app.main import app + from app.routers.v2 import curated + + def _override(): + yield db + + app.dependency_overrides[curated.get_db] = _override + yield client + app.dependency_overrides.pop(curated.get_db, None) + + +def _login(client, username, password): + r = client.post("/api/v1/auth/login", json={"username": username, "password": password}) + return {"Authorization": f"Bearer {r.json()['data']['access_token']}"} + + +def _make_curated_with_media(db) -> tuple[str, list[str]]: + ds = Dataset(id=str(uuid.uuid4()), name=f"del-{uuid.uuid4().hex[:6]}", kind="curated") + db.add(ds) + db.flush() + ver = DatasetVersion(id=str(uuid.uuid4()), dataset_id=ds.id, version_no=1) + db.add(ver) + db.flush() + uris = [f"s3://curated/{uuid.uuid4().hex}.pdf", f"s3://curated/{uuid.uuid4().hex}.docx"] + media = MediaItem( + id=str(uuid.uuid4()), dataset_version_id=ver.id, media_type="pdf", + storage_uri=uris[0], ocr_result_uri=uris[1], + ) + db.add(media) + db.commit() + return ds.id, uris + + +def test_delete_curated_cleans_up_minio_objects(curated_client, db, admin_user, monkeypatch): + """Deleting a curated dataset must remove its MediaItem MinIO objects + (storage_uri and ocr_result_uri) — previously nothing in the delete path + ever called StorageService.delete_object, leaving them as permanent + orphaned storage.""" + ds_id, expected_uris = _make_curated_with_media(db) + deleted = [] + + class FakeStorage: + def delete_object(self, uri): + deleted.append(uri) + + monkeypatch.setattr("app.services.storage_service.get_storage_service", lambda: FakeStorage()) + + headers = _login(curated_client, "admin", "admin123") + r = curated_client.delete(f"/api/v2/curated/{ds_id}", headers=headers) + assert r.status_code == 204 + assert sorted(deleted) == sorted(expected_uris) + + +def test_delete_curated_removes_legacy_mirror_row(curated_client, db, admin_user, monkeypatch): + """Once a dataset has been through review, _ensure_curated_dataset_row + (and the older /review endpoint) lazily creates a v2_curated_datasets + row with the SAME id as the real v2_datasets row, purely as an FK bridge + for CuratedReview. Deleting the dataset must remove that mirror row too + — previously delete_curated only ever deleted the v2_datasets row and + the CuratedReview rows, leaving the CuratedDataset mirror as a permanent + orphan.""" + monkeypatch.setattr("app.services.storage_service.get_storage_service", lambda: object()) + ds_id, _ = _make_curated_with_media(db) + # simulate what a completed review leaves behind + db.add(CuratedDataset(id=ds_id, name="mirror", status="approved")) + db.add(CuratedReview( + id=str(uuid.uuid4()), curated_dataset_id=ds_id, status="approved", + decided_at=datetime.now(timezone.utc), + )) + db.commit() + + headers = _login(curated_client, "admin", "admin123") + r = curated_client.delete(f"/api/v2/curated/{ds_id}", headers=headers) + assert r.status_code == 204 + + assert db.query(CuratedDataset).filter(CuratedDataset.id == ds_id).first() is None + assert db.query(CuratedReview).filter(CuratedReview.curated_dataset_id == ds_id).first() is None + + +def test_delete_curated_survives_minio_failure(curated_client, db, admin_user, monkeypatch): + """MinIO cleanup is best-effort: a Postgres delete that already committed + must still return 204 even if object storage cleanup blows up.""" + ds_id, _ = _make_curated_with_media(db) + + class BrokenStorage: + def delete_object(self, uri): + raise ConnectionError("minio unreachable") + + monkeypatch.setattr("app.services.storage_service.get_storage_service", lambda: BrokenStorage()) + + headers = _login(curated_client, "admin", "admin123") + r = curated_client.delete(f"/api/v2/curated/{ds_id}", headers=headers) + assert r.status_code == 204 diff --git a/backend/tests/v2/test_logic_actions_runtime.py b/backend/tests/v2/test_logic_actions_runtime.py index 4ada5abc..98cc2369 100644 --- a/backend/tests/v2/test_logic_actions_runtime.py +++ b/backend/tests/v2/test_logic_actions_runtime.py @@ -2,6 +2,8 @@ from app.models.action import Action from app.models.logic import LogicRule from app.models.ontology import OntologyProject +from app.models.user import User +from app.services.ontology_access import creator_grant from app.models.v2.logic import OntologyLogicRule from app.models.v2.action import OntologyActionRun, OntologyActionType from app.routers.actions import publish_actions @@ -203,6 +205,8 @@ def test_action_runtime_rejects_missing_required_parameter(db): def test_v1_logic_publish_syncs_v2_status(db): ontology_id = "ont-runtime-5" + actor = User(id="runtime", username="runtime", email="runtime@example.test", password_hash="unused", role="editor") + db.add(actor) db.add(OntologyProject(id=ontology_id, name=ontology_id, domain="test", created_by="runtime", security_domain_id="00000000-0000-0000-0000-000000000001")) db.add(LogicRule( id="logic-v1", @@ -224,9 +228,10 @@ def test_v1_logic_publish_syncs_v2_status(db): status="draft", )) db.commit() + creator_grant(db, ontology_id, actor) with pytest.raises(HTTPException) as exc: - publish_logic_rules(ontology_id, db) + publish_logic_rules(ontology_id, db, actor) assert "PUBLICATION_NOT_ENABLED" in str(exc.value.detail) # direct tool publication is closed; nothing is published assert db.query(LogicRule).filter_by(id="logic-v1").first().status == "draft" @@ -235,6 +240,8 @@ def test_v1_logic_publish_syncs_v2_status(db): def test_v1_action_publish_syncs_v2_status(db): ontology_id = "ont-runtime-6" + actor = User(id="runtime", username="runtime", email="runtime@example.test", password_hash="unused", role="editor") + db.add(actor) db.add(OntologyProject(id=ontology_id, name=ontology_id, domain="test", created_by="runtime", security_domain_id="00000000-0000-0000-0000-000000000001")) db.add(Action( id="action-v1", @@ -254,9 +261,10 @@ def test_v1_action_publish_syncs_v2_status(db): status="draft", )) db.commit() + creator_grant(db, ontology_id, actor) with pytest.raises(HTTPException) as exc: - publish_actions(ontology_id, db) + publish_actions(ontology_id, db, actor) assert "PUBLICATION_NOT_ENABLED" in str(exc.value.detail) # direct tool publication is closed; nothing is published assert db.query(Action).filter_by(id="action-v1").first().status == "draft" diff --git a/backend/tests/v2/test_pipeline_connector_safety.py b/backend/tests/v2/test_pipeline_connector_safety.py new file mode 100644 index 00000000..c5180ce3 --- /dev/null +++ b/backend/tests/v2/test_pipeline_connector_safety.py @@ -0,0 +1,195 @@ +"""Builder pipeline connector safety contracts.""" + +import uuid + +import pytest + +from app.models.v2.pipeline import Pipeline + + +@pytest.fixture +def pipelines_client(client, db): + """Use the test session for the v2 pipeline router.""" + from app.main import app + from app.routers.v2 import pipelines + + def _override(): + yield db + + app.dependency_overrides[pipelines.get_db] = _override + yield client + app.dependency_overrides.pop(pipelines.get_db, None) + + +def _headers(client, username="admin", password="admin123"): + response = client.post( + "/api/v1/auth/login", + json={"username": username, "password": password}, + ) + return {"Authorization": f"Bearer {response.json()['data']['access_token']}"} + + +def _inline_definition(source_type: str) -> dict: + return { + "nodes": [{ + "id": "source", + "type": "connector", + "config": { + "source_type": source_type, + "config_values": { + "uri": "mongodb://user:secret@example.invalid/orders" + if source_type == "mongodb" + else "https://example.invalid/orders", + "headers": '{"Authorization":"Bearer secret"}', + }, + }, + }], + "edges": [], + } + + +@pytest.mark.parametrize("source_type", ["mongodb", "rest_api"]) +def test_create_rejects_unsupported_inline_connectors_before_persistence( + pipelines_client, db, admin_user, source_type, +): + name = f"unsupported-{source_type}-{uuid.uuid4().hex}" + + response = pipelines_client.post( + "/api/v2/pipelines", + json={"name": name, "definition": _inline_definition(source_type)}, + headers=_headers(pipelines_client), + ) + + assert response.status_code == 422 + assert response.json()["detail"] == "UNSUPPORTED_INLINE_CONNECTOR" + assert db.query(Pipeline).filter(Pipeline.name == name).count() == 0 + + +def test_update_rejects_unsupported_inline_connector_before_mutating_definition( + pipelines_client, db, admin_user, +): + pipeline = Pipeline( + name=f"existing-{uuid.uuid4().hex}", + route="A", + spec={}, + definition={"nodes": [], "edges": []}, + ) + db.add(pipeline) + db.commit() + pipeline_id = pipeline.id + + response = pipelines_client.put( + f"/api/v2/pipelines/{pipeline_id}", + json={"definition": _inline_definition("rest_api")}, + headers=_headers(pipelines_client), + ) + + assert response.status_code == 422 + assert response.json()["detail"] == "UNSUPPORTED_INLINE_CONNECTOR" + assert db.get(Pipeline, pipeline_id).definition == {"nodes": [], "edges": []} + + +def test_validate_marks_legacy_unsupported_connector_definition_invalid( + pipelines_client, db, admin_user, +): + pipeline = Pipeline( + name=f"legacy-{uuid.uuid4().hex}", + route="A", + spec={}, + definition=_inline_definition("mongodb"), + ) + db.add(pipeline) + db.commit() + + response = pipelines_client.post( + f"/api/v2/pipelines/{pipeline.id}/validate", + headers=_headers(pipelines_client), + ) + + assert response.status_code == 200 + payload = response.json() + assert payload["valid"] is False + assert payload["errors"] == [{ + "node_id": "source", + "severity": "error", + "code": "UNSUPPORTED_INLINE_CONNECTOR", + "message": "MongoDB/REST inline connector nodes are unsupported in Builder; use File Upload in Builder and manage external connections in Data Connections.", + }] + + +def test_legacy_inline_connector_secrets_are_not_returned_by_pipeline_detail( + pipelines_client, db, admin_user, +): + pipeline = Pipeline( + name=f"legacy-secret-{uuid.uuid4().hex}", + route="A", + spec={}, + definition=_inline_definition("rest_api"), + ) + db.add(pipeline) + db.commit() + + response = pipelines_client.get( + f"/api/v2/pipelines/{pipeline.id}", + headers=_headers(pipelines_client), + ) + + assert response.status_code == 200 + assert response.json()["definition"]["nodes"][0]["config"] == {"source_type": "rest_api"} + assert "Bearer secret" not in response.text + + +def test_legacy_inline_connector_cannot_queue_a_run(pipelines_client, db, admin_user): + pipeline = Pipeline( + name=f"legacy-run-{uuid.uuid4().hex}", + route="A", + spec={}, + definition=_inline_definition("mongodb"), + status="draft", + ) + db.add(pipeline) + db.commit() + + response = pipelines_client.post( + f"/api/v2/pipelines/{pipeline.id}/run", + headers=_headers(pipelines_client), + ) + + assert response.status_code == 422 + assert response.json()["detail"] == "UNSUPPORTED_INLINE_CONNECTOR" + db.refresh(pipeline) + assert pipeline.status == "draft" + + +def test_saved_sql_source_does_not_store_legacy_inline_credentials( + pipelines_client, db, admin_user, +): + definition = { + "nodes": [{ + "id": "sql-source", "type": "connector", + "config": { + "source_type": "postgresql", + "connection_id": "saved-connection", + "resource": "orders", + "config_values": { + "uri": "postgresql://user:secret@example.invalid/orders", + "headers": {"Authorization": "Bearer secret"}, + }, + }, + }], + "edges": [], + } + response = pipelines_client.post( + "/api/v2/pipelines", + json={"name": f"safe-sql-{uuid.uuid4().hex}", "definition": definition}, + headers=_headers(pipelines_client), + ) + + assert response.status_code == 201 + stored = db.get(Pipeline, response.json()["id"]).definition + assert stored["nodes"][0]["config"] == { + "source_type": "postgresql", + "connection_id": "saved-connection", + "resource": "orders", + } + assert "secret" not in response.text diff --git a/backend/tests/v2/test_upload_limits.py b/backend/tests/v2/test_upload_limits.py index 9fab92fe..be5551ed 100644 --- a/backend/tests/v2/test_upload_limits.py +++ b/backend/tests/v2/test_upload_limits.py @@ -2,6 +2,7 @@ import io import pytest +from fastapi import HTTPException from app.config import settings @@ -45,6 +46,36 @@ def test_upload_rejects_oversized_file(datasets_client, admin_user, monkeypatch) assert r.status_code == 413 +@pytest.mark.asyncio +async def test_bounded_upload_read_stops_after_limit_plus_one_byte(monkeypatch): + from app.routers.v2.datasets import _read_upload_bounded + + limit = 8 + payload = b"x" * (limit + 1000) + + class TrackedUpload: + def __init__(self, value): + self.value = value + self.offset = 0 + self.read_sizes = [] + + async def read(self, size=-1): + self.read_sizes.append(size) + end = min(self.offset + size, len(self.value)) + chunk = self.value[self.offset:end] + self.offset = end + return chunk + + upload = TrackedUpload(payload) + + with pytest.raises(HTTPException) as error: + await _read_upload_bounded(upload, limit, chunk_size=4) + + assert error.value.status_code == 413 + assert upload.offset == limit + 1 + assert upload.read_sizes == [4, 4, 1] + + def test_upload_accepts_normal_csv(datasets_client, admin_user): r = datasets_client.post( "/api/v2/datasets/upload", diff --git a/docker-compose.v2.yml b/docker-compose.v2.yml index a5f66c9d..85c422da 100644 --- a/docker-compose.v2.yml +++ b/docker-compose.v2.yml @@ -78,6 +78,9 @@ services: build: ./backend ports: ["8000:8000"] env_file: .env + # In-network clients use the container port, not the host's 8001 mapping. + environment: &chroma-environment + CHROMA_PORT: "8000" depends_on: migration: condition: service_completed_successfully @@ -103,6 +106,7 @@ services: env_file: .env profiles: ["refresh"] environment: + <<: *chroma-environment REFRESH_WORKER_CONCURRENCY: ${REFRESH_WORKER_CONCURRENCY:-2} REFRESH_WORKER_PREFETCH_MULTIPLIER: ${REFRESH_WORKER_PREFETCH_MULTIPLIER:-1} REFRESH_SOFT_TIME_LIMIT_SECONDS: ${REFRESH_SOFT_TIME_LIMIT_SECONDS:-300} @@ -128,6 +132,7 @@ services: env_file: .env profiles: ["refresh"] environment: + <<: *chroma-environment REFRESH_WORKER_CONCURRENCY: ${REFRESH_WORKER_CONCURRENCY:-2} REFRESH_WORKER_PREFETCH_MULTIPLIER: ${REFRESH_WORKER_PREFETCH_MULTIPLIER:-1} REFRESH_SOFT_TIME_LIMIT_SECONDS: ${REFRESH_SOFT_TIME_LIMIT_SECONDS:-300} @@ -153,6 +158,7 @@ services: env_file: .env profiles: ["refresh"] environment: + <<: *chroma-environment REFRESH_WORKER_CONCURRENCY: ${REFRESH_WORKER_CONCURRENCY:-2} REFRESH_WORKER_PREFETCH_MULTIPLIER: ${REFRESH_WORKER_PREFETCH_MULTIPLIER:-1} REFRESH_SOFT_TIME_LIMIT_SECONDS: ${REFRESH_SOFT_TIME_LIMIT_SECONDS:-300} @@ -178,6 +184,7 @@ services: env_file: .env profiles: ["refresh"] environment: + <<: *chroma-environment REFRESH_WORKER_CONCURRENCY: ${REFRESH_WORKER_CONCURRENCY:-2} REFRESH_WORKER_PREFETCH_MULTIPLIER: ${REFRESH_WORKER_PREFETCH_MULTIPLIER:-1} REFRESH_SOFT_TIME_LIMIT_SECONDS: ${REFRESH_SOFT_TIME_LIMIT_SECONDS:-300} @@ -202,6 +209,7 @@ services: build: ./backend env_file: .env profiles: ["artifact"] + environment: *chroma-environment depends_on: migration: condition: service_completed_successfully @@ -220,6 +228,7 @@ services: build: ./backend env_file: .env profiles: ["agent"] + environment: *chroma-environment depends_on: migration: condition: service_completed_successfully @@ -238,6 +247,7 @@ services: build: ./backend env_file: .env profiles: ["housekeeping"] + environment: *chroma-environment depends_on: migration: condition: service_completed_successfully @@ -255,6 +265,7 @@ services: celery_beat: build: ./backend env_file: .env + environment: *chroma-environment depends_on: migration: condition: service_completed_successfully diff --git a/frontend/src/api/client.test.ts b/frontend/src/api/client.test.ts new file mode 100644 index 00000000..2e870e94 --- /dev/null +++ b/frontend/src/api/client.test.ts @@ -0,0 +1,42 @@ +import { afterAll, afterEach, beforeAll, expect, it, vi } from 'vitest' +import { http, HttpResponse } from 'msw' +import { setupServer } from 'msw/node' +import { apiClient, apiClientV2 } from './client' +import { useAuthStore } from '@/stores/authStore' + +const server = setupServer() +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })) +afterAll(() => server.close()) +afterEach(() => { + server.resetHandlers() + vi.restoreAllMocks() + useAuthStore.setState({ token: null, user: null }) +}) + +it.each([['v1', apiClient], ['v2', apiClientV2]] as const)('%s permission denial preserves login without refreshing', async (version, client) => { + useAuthStore.setState({ token: 'valid-session' }) + const fetch = vi.spyOn(globalThis, 'fetch') + server.use(http.get(`*/api/${version}/denied`, () => HttpResponse.json({ detail: 'Forbidden' }, { status: 403 }))) + await expect(client.get('/denied')).rejects.toMatchObject({ detail: 'Forbidden' }) + expect(useAuthStore.getState().token).toBe('valid-session') + expect(fetch).not.toHaveBeenCalled() +}) + +it('refreshes an expired session once and retries with the new token', async () => { + useAuthStore.setState({ token: 'expired' }) + const fetch = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(JSON.stringify({ data: { access_token: 'renewed' } }), { status: 200 })) + server.use(http.get('*/api/v1/protected', ({ request }) => request.headers.get('Authorization') === 'Bearer renewed' + ? HttpResponse.json({ data: { ok: true } }) : HttpResponse.json({}, { status: 401 }))) + await expect(apiClient.get('/protected')).resolves.toEqual({ ok: true }) + expect(useAuthStore.getState().token).toBe('renewed') + expect(fetch).toHaveBeenCalledTimes(1) +}) + +it('clears login if the refreshed token is still rejected', async () => { + useAuthStore.setState({ token: 'expired' }) + const fetch = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(JSON.stringify({ data: { access_token: 'rejected' } }), { status: 200 })) + server.use(http.get('*/api/v1/protected', () => HttpResponse.json({ detail: 'Unauthorized' }, { status: 401 }))) + await expect(apiClient.get('/protected')).rejects.toMatchObject({ detail: 'Unauthorized' }) + expect(useAuthStore.getState().token).toBeNull() + expect(fetch).toHaveBeenCalledTimes(1) +}) diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 2e7b5700..ea6b5e1e 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -65,7 +65,7 @@ function createApiClient(baseURL: string, options?: { runtimeDelegation?: boolea return client(config) } } - if (!options?.runtimeDelegation && !options?.preserveSessionOnAuthorizationFailure && (status === 401 || status === 403)) { + if (!options?.runtimeDelegation && !options?.preserveSessionOnAuthorizationFailure && status === 401) { useAuthStore.getState().logout() } if (options?.runtimeDelegation && status === 401) { diff --git a/frontend/src/i18n/en.json b/frontend/src/i18n/en.json index 6ab298cf..9b6e9412 100644 --- a/frontend/src/i18n/en.json +++ b/frontend/src/i18n/en.json @@ -124,8 +124,8 @@ "confirm_delete_msg": "Delete ontology \"{{name}}\"? This cannot be undone.", "cancel": "Cancel", "confirm": "Delete", - "delete_failed_bound_agent": "Cannot delete: this ontology is bound to an Agent — unbind it first", - "delete_failed_dependent_data": "Cannot delete: this ontology has dependent execution records (runs, sandbox simulations, etc.) — clean those up first", + "delete_failed_bound_agent": "Cannot delete: this ontology is bound to an Agent version, and Agent version bindings can never be changed once created — so this ontology cannot be deleted", + "delete_failed_dependent_data": "Cannot delete: this ontology has published releases with dependent execution records (runs, sandbox simulations, etc.) that are kept permanently for audit purposes — so this ontology cannot be deleted", "delete_failed_generic": "Delete failed, please try again", "deleting": "Deleting…", "select_all": "Select all", @@ -247,7 +247,6 @@ "delete_title": "Delete Rule", "delete_confirm": "Delete rule “{{name}}”? This cannot be undone.", "ph_search": "Search rule name / definition…", - "publish_all": "Publish All", "col_name": "Name", "col_definition": "Definition", "col_desc": "Description", @@ -263,7 +262,6 @@ "empty": "No actions yet", "delete_title": "Delete Action", "delete_confirm": "Delete action “{{name}}”? This cannot be undone.", - "publish_all": "Publish All", "col_name": "Name", "col_rule": "Rule", "col_desc": "Description", @@ -328,6 +326,8 @@ "empty": "No models configured. Click \"Add Model\" to get started.", "confirm_delete": "Confirm Delete", "confirm_delete_msg": "Delete model \"{{name}}\"?", + "delete_failed_referenced": "Cannot delete: this model is still used by an Agent. Unbind or delete the Agent first.", + "delete_failed_generic": "Delete failed, please try again", "config_type_llm": "LLM Config", "config_type_ocr": "OCR Config", "config_type_other": "Other Config", @@ -565,6 +565,7 @@ "tools": { "available_later": "Available later", "bind": "Bind", + "blocked_hint": "Your account has no data grant (read schema, instances and relations) for: {{names}}. Ask an admin to grant data access.", "bound": "Bound", "capabilities": "Capabilities", "external": "External Tools", @@ -1306,7 +1307,15 @@ "test_failed": "Failed", "test_testing": "Testing...", "test_connection": "Test Connection", - "sync_mode_label": "Sync Mode" + "sync_mode_label": "Sync Mode", + "saved_connection_label": "Saved Connection", + "source_table_label": "Source Table", + "select_connection": "Select a saved connection", + "select_table": "Select a table", + "loading_connections": "Loading connections...", + "loading_resources": "Loading tables...", + "no_saved_connections": "Create a database connection in Data Connections first", + "resources_failed": "Could not discover tables" }, "outputInspector": { "output_type_label": "Output Type", @@ -1411,7 +1420,16 @@ "drop_release": "Release to add files", "drop_hint": "Drag files here, or", "drop_hint_click": "click to select", - "drop_formats_hint": "Supports CSV, XLSX, JSON, PDF, DOCX and more, multiple selection allowed" + "drop_formats_hint": "Supports CSV, XLSX, JSON, PDF, DOCX and more, multiple selection allowed", + "test_connection": "Test Connection", + "testing": "Testing...", + "test_success": "Connection succeeded", + "test_failed": "Connection failed", + "discover_tables": "Discover Tables", + "hide_tables": "Hide Tables", + "no_tables": "No tables found", + "resources_failed": "Could not discover tables", + "pipeline_sync_hint": "Run a selected table from Pipeline Builder" }, "curatedReview": { "notes_ph": "Review notes (optional)", diff --git a/frontend/src/i18n/zh.json b/frontend/src/i18n/zh.json index ad9a4afe..29ca657f 100644 --- a/frontend/src/i18n/zh.json +++ b/frontend/src/i18n/zh.json @@ -124,8 +124,8 @@ "confirm_delete_msg": "确定要删除本体「{{name}}」吗?此操作不可撤销。", "cancel": "取消", "confirm": "确认删除", - "delete_failed_bound_agent": "无法删除:该本体已绑定至智能体,请先解绑对应智能体后再删除", - "delete_failed_dependent_data": "无法删除:该本体存在关联的执行记录(如运行历史、沙盒模拟等),请先清理后再删除", + "delete_failed_bound_agent": "无法删除:该本体已被某个智能体版本绑定,智能体版本一经创建绑定即不可更改,因此这个本体无法删除", + "delete_failed_dependent_data": "无法删除:该本体存在已发布版本及关联的执行记录(运行历史、沙盒模拟等),出于审计要求这些记录会永久保留,因此本体无法删除", "delete_failed_generic": "删除失败,请稍后重试", "deleting": "删除中…", "select_all": "全选", @@ -247,7 +247,6 @@ "delete_title": "删除规则", "delete_confirm": "确定删除规则「{{name}}」吗?此操作不可撤销。", "ph_search": "搜索规则名称 / 定义…", - "publish_all": "发布全部", "col_name": "名称", "col_definition": "定义", "col_desc": "描述", @@ -263,7 +262,6 @@ "empty": "暂无动作", "delete_title": "删除动作", "delete_confirm": "确定删除动作「{{name}}」吗?此操作不可撤销。", - "publish_all": "发布全部", "col_name": "名称", "col_rule": "规则", "col_desc": "描述", @@ -328,6 +326,8 @@ "empty": "暂无模型配置,点击「添加模型」开始", "confirm_delete": "确认删除", "confirm_delete_msg": "删除模型「{{name}}」?", + "delete_failed_referenced": "无法删除:该模型仍被智能体使用,请先解绑或删除相关智能体后再删除", + "delete_failed_generic": "删除失败,请稍后重试", "config_type_llm": "LLM配置", "config_type_ocr": "OCR配置", "config_type_other": "其他配置", @@ -565,6 +565,7 @@ "tools": { "available_later": "后续提供", "bind": "绑定", + "blocked_hint": "当前账号没有以下本体的数据授权(读取结构、实例和关系):{{names}}。请联系管理员开通数据授权。", "bound": "已绑定", "capabilities": "能力", "external": "外部工具", @@ -1306,7 +1307,15 @@ "test_failed": "失败", "test_testing": "测试中...", "test_connection": "测试连接", - "sync_mode_label": "同步模式" + "sync_mode_label": "同步模式", + "saved_connection_label": "已保存连接", + "source_table_label": "源数据表", + "select_connection": "选择已保存的连接", + "select_table": "选择数据表", + "loading_connections": "正在加载连接...", + "loading_resources": "正在加载数据表...", + "no_saved_connections": "请先在数据连接中创建数据库连接", + "resources_failed": "无法发现数据表" }, "outputInspector": { "output_type_label": "输出类型", @@ -1411,7 +1420,16 @@ "drop_release": "松开以添加文件", "drop_hint": "拖拽文件到此处,或", "drop_hint_click": "点击选择", - "drop_formats_hint": "支持 CSV、XLSX、JSON、PDF、DOCX 等格式,可多选" + "drop_formats_hint": "支持 CSV、XLSX、JSON、PDF、DOCX 等格式,可多选", + "test_connection": "测试连接", + "testing": "测试中...", + "test_success": "连接成功", + "test_failed": "连接失败", + "discover_tables": "发现数据表", + "hide_tables": "收起数据表", + "no_tables": "未发现数据表", + "resources_failed": "无法发现数据表", + "pipeline_sync_hint": "请在 Pipeline Builder 中选择数据表并运行" }, "curatedReview": { "notes_ph": "审核备注(可选)", diff --git a/frontend/src/pages/agents/application/ConversationPanel.test.tsx b/frontend/src/pages/agents/application/ConversationPanel.test.tsx index 692c6eb8..8153f06b 100644 --- a/frontend/src/pages/agents/application/ConversationPanel.test.tsx +++ b/frontend/src/pages/agents/application/ConversationPanel.test.tsx @@ -14,6 +14,20 @@ function renderPanel(messages: AgentMessage[], stream: StreamState = initialStre } describe('P4A-CONVERSATION', () => { + it('hides prior-turn evidence synchronously when the turn changes', async () => { + const get = vi.spyOn(apiClient, 'get').mockImplementation(url => url.includes('/old/') + ? Promise.resolve({ items: [{ sequence: 1, event_type: 'final_response', payload: { audit_event_id: 'old-audit' } }] }) as never + : new Promise(() => {})) + const props = { messages: [], stream: { ...initialStreamState, terminal: true }, clarification: null, + pendingApprovalId: null, onSend: () => {}, onAnswerClarification: () => {}, onApprovalResolved: () => {}, onRetry: () => {} } + try { + const view = render() + await screen.findByText('old-audit') + view.rerender() + expect(screen.queryByTestId('journey-evidence')).toBeNull() + } finally { get.mockRestore() } + }) + it('strips the legacy canned "Answer for …" prefix from persisted messages', () => { const messages: AgentMessage[] = [{ id: 'm-1', session_id: 's-1', turn_id: 't-1', role: 'assistant', diff --git a/frontend/src/pages/agents/application/ConversationPanel.tsx b/frontend/src/pages/agents/application/ConversationPanel.tsx index ab7d5b23..b81ee299 100644 --- a/frontend/src/pages/agents/application/ConversationPanel.tsx +++ b/frontend/src/pages/agents/application/ConversationPanel.tsx @@ -88,7 +88,8 @@ export default function ConversationPanel({ const { t } = useTranslation() const [draft, setDraft] = useState('') const [clarificationAnswer, setClarificationAnswer] = useState('') - const [journeyEvents, setJourneyEvents] = useState([]) + const [journey, setJourney] = useState<{ turnId: string; events: RuntimeEventRecord[] } | null>(null) + const journeyEvents = journey?.turnId === turnId ? journey.events : [] const [showGovernedPlan, setShowGovernedPlan] = useState(false) // Persisted turn events back the always-visible business-journey @@ -98,11 +99,10 @@ export default function ConversationPanel({ // a terminal SSE/polling state. useEffect(() => { let cancelled = false - void Promise.resolve().then(() => { if (!cancelled) setJourneyEvents([]) }) if (!turnId) return () => { cancelled = true } const fetchEvents = () => { apiClient.get<{ items: RuntimeEventRecord[] }>(`/agent-turns/${turnId}/events?limit=100`) - .then(res => { if (!cancelled) setJourneyEvents(Array.isArray(res.items) ? res.items : []) }) + .then(res => { if (!cancelled) setJourney({ turnId, events: Array.isArray(res.items) ? res.items : [] }) }) .catch(() => {}) } fetchEvents() @@ -151,20 +151,20 @@ export default function ConversationPanel({ // `app.services.runtime.answer_stream`) — polled on its own fast cadence // rather than piggybacking on the slower journeyEvents poll above, since // this is what actually needs to feel live. - const [liveAnswerText, setLiveAnswerText] = useState('') + const [liveAnswer, setLiveAnswer] = useState<{ turnId: string; text: string } | null>(null) useEffect(() => { - setLiveAnswerText('') if (!turnId || !isThinking) return let cancelled = false const poll = () => { apiClient.get<{ text: string } | null>(`/agent-turns/${turnId}/answer-stream`) - .then(res => { if (!cancelled && res) setLiveAnswerText(res.text) }) + .then(res => { if (!cancelled && res) setLiveAnswer({ turnId, text: res.text }) }) .catch(() => {}) } poll() const interval = setInterval(poll, ANSWER_STREAM_POLL_INTERVAL_MS) return () => { cancelled = true; clearInterval(interval) } }, [turnId, isThinking]) + const liveAnswerText = isThinking && liveAnswer?.turnId === turnId ? liveAnswer.text : '' const submit = () => { if (!draft.trim()) return diff --git a/frontend/src/pages/agents/detail/ToolConfigTab.test.tsx b/frontend/src/pages/agents/detail/ToolConfigTab.test.tsx index f6254cbd..b353db02 100644 --- a/frontend/src/pages/agents/detail/ToolConfigTab.test.tsx +++ b/frontend/src/pages/agents/detail/ToolConfigTab.test.tsx @@ -207,6 +207,22 @@ describe('P2C-TOOLS', () => { expect(bindings[0].enabled_categories).not.toContain('logic') }) + it('clears a stale conflict banner once the ontology is unbound', async () => { + toolsHandlers() + server.use( + http.post('*/api/v1/agents/a-1/tool-validation', () => + HttpResponse.json({ data: { valid: false, blocked: ['o-1'], capabilities: [] }, message: 'ok' })), + ) + renderTab() + await screen.findByTestId('ontology-picker') + await pickOntology('Supply Ontology') + expect(await screen.findByText('绑定配置存在冲突')).toBeTruthy() + // the blocked ontology is named, with the actual fix (a data grant), not a bare id + expect(screen.getByText(/没有以下本体的数据授权.*Supply Ontology.*联系管理员/)).toBeTruthy() + await userEvent.click(screen.getByRole('button', { name: '解绑' })) + await waitFor(() => expect(screen.queryByText('绑定配置存在冲突')).toBeNull()) + }) + it('binds at most one ontology: the picker locks once bound, unbind to switch', async () => { toolsHandlers() server.use( diff --git a/frontend/src/pages/agents/detail/ToolConfigTab.tsx b/frontend/src/pages/agents/detail/ToolConfigTab.tsx index 49ecf906..d7973a48 100644 --- a/frontend/src/pages/agents/detail/ToolConfigTab.tsx +++ b/frontend/src/pages/agents/detail/ToolConfigTab.tsx @@ -147,7 +147,11 @@ export default function ToolConfigTab({ agentId, activeVersion, canEdit, onSaved // validate the proposed binding set whenever it changes useEffect(() => { - if (!canEdit || bindings.length === 0) return + if (!canEdit || bindings.length === 0) { + // drop a stale result so an old conflict banner doesn't outlive the binding + void Promise.resolve().then(() => setValidation(null)) + return + } let cancelled = false agentToolsApi.validateAgentTools(agentId, { ontology_ids: bindings.map(b => b.ontology_id) }) .then(result => { if (!cancelled) setValidation(result) }) @@ -243,7 +247,11 @@ export default function ToolConfigTab({ agentId, activeVersion, canEdit, onSaved : t('agent.tools.invalid', '绑定配置存在冲突')}

{validation.blocked && validation.blocked.length > 0 && ( -

{validation.blocked.join(', ')}

+

+ {t('agent.tools.blocked_hint', { + names: validation.blocked.map(id => ontologies.find(o => o.id === id)?.name ?? id).join(', '), + })} +

)} - + @@ -274,7 +279,9 @@ export default function ModelsPage() { )} deleteTarget && deleteMut.mutate(deleteTarget.id)} onCancel={() => setDeleteTarget(null)} /> + error={deleteError} + onConfirm={() => deleteTarget && deleteMut.mutate(deleteTarget.id)} + onCancel={() => { setDeleteTarget(null); setDeleteError('') }} /> ) } diff --git a/frontend/src/pages/ontologies/detail/OntologyDetailPage.tsx b/frontend/src/pages/ontologies/detail/OntologyDetailPage.tsx index ac427570..1071a176 100644 --- a/frontend/src/pages/ontologies/detail/OntologyDetailPage.tsx +++ b/frontend/src/pages/ontologies/detail/OntologyDetailPage.tsx @@ -1,6 +1,6 @@ -import React, { useState, lazy, Suspense } from 'react' +import React, { useState, useEffect, lazy, Suspense } from 'react' import { useParams, useNavigate, useSearchParams } from 'react-router-dom' -import { useQuery, useQueryClient } from '@tanstack/react-query' +import { useIsMutating, useQuery, useQueryClient } from '@tanstack/react-query' import { useTranslation } from 'react-i18next' import { ontologyApi } from '@/api/ontologies' import StatusBadge from '@/components/StatusBadge' @@ -63,6 +63,13 @@ export default function OntologyDetailPage() { const qc = useQueryClient() const [localStatus, setLocalStatus] = useState(null) + // tab edits mark the working copy dirty server-side — refetch the ontology + // once they settle so the "publish new version" button can appear + const mutating = useIsMutating() + useEffect(() => { + if (mutating === 0) qc.invalidateQueries({ queryKey: ['ontology', id] }) + }, [mutating, qc, id]) + if (isLoading) return
{t('common.loading')}
if (!ontology) return
Ontology not found
@@ -79,6 +86,7 @@ export default function OntologyDetailPage() { const invalidateList = () => { qc.invalidateQueries({ queryKey: ['ontologies'] }) qc.invalidateQueries({ queryKey: ['stats'] }) + qc.invalidateQueries({ queryKey: ['ontology', id] }) } const isPipelineMode = ontology.build_mode === 'pipeline_mapping' @@ -105,7 +113,7 @@ export default function OntologyDetailPage() {
-
diff --git a/frontend/src/pages/ontologies/detail/tabs/ActionsTab.tsx b/frontend/src/pages/ontologies/detail/tabs/ActionsTab.tsx index 8a23e603..8b526120 100644 --- a/frontend/src/pages/ontologies/detail/tabs/ActionsTab.tsx +++ b/frontend/src/pages/ontologies/detail/tabs/ActionsTab.tsx @@ -6,7 +6,7 @@ import { useTranslation } from 'react-i18next' import { ontologyApi } from '@/api/ontologies' import { apiClient } from '@/api/client' import ConfirmDialog from '@/components/ConfirmDialog' -import { Pencil, Trash2, Plus, ToggleLeft, ToggleRight, CheckCircle, Loader2 } from 'lucide-react' +import { Pencil, Trash2, Plus, ToggleLeft, ToggleRight } from 'lucide-react' import type { Action } from '@/types/ontology' type ActionRow = Action @@ -36,20 +36,11 @@ export default function ActionsTab({ ontologyId }: { ontologyId: string }) { mutationFn: (id: string) => apiClient.post(`/ontologies/${ontologyId}/actions/${id}/toggle`), onSuccess: () => qc.invalidateQueries({ queryKey: ['actions', ontologyId] }), }) - const publishMut = useMutation({ - mutationFn: () => apiClient.post(`/ontologies/${ontologyId}/actions/publish`), - onSuccess: () => qc.invalidateQueries({ queryKey: ['actions', ontologyId] }), - }) return (
-
-
)} - {sourceType !== 'file' && (
{DB_CONFIG_FIELDS[sourceType]?.map(f => (
| undefined)?.[f.key] || '')} onChange={e => { const cv2 = { ...((config.config_values as Record | undefined) || {}), [f.key]: e.target.value }; onChange('config_values', cv2); setTestStatus('idle') }} placeholder={f.placeholder} className="w-full border rounded-lg px-3 py-1.5 text-sm" />
))}
)} + {isSqlSource && (
+
+ + + {!connectionLoading && matchingConnections.length === 0 &&

{t('connectorInspector.no_saved_connections')}

} +
+
+ + + {resourceError &&

{resourceError}

} +
+
)} + {sourceType !== 'file' && !isSqlSource && !isUnsupportedSource && (
{DB_CONFIG_FIELDS[sourceType]?.map(f => (
| undefined)?.[f.key] || '')} onChange={e => { const cv2 = { ...((config.config_values as Record | undefined) || {}), [f.key]: e.target.value }; onChange('config_values', cv2); setTestStatus('idle') }} placeholder={f.placeholder} className="w-full border rounded-lg px-3 py-1.5 text-sm" />
))}
)}
-
-
+
) } diff --git a/frontend/src/pages/pipelines/connections/ConnectionsTab.test.tsx b/frontend/src/pages/pipelines/connections/ConnectionsTab.test.tsx new file mode 100644 index 00000000..b681a30a --- /dev/null +++ b/frontend/src/pages/pipelines/connections/ConnectionsTab.test.tsx @@ -0,0 +1,70 @@ +import '@/i18n' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { render, screen, fireEvent, waitFor } from '@testing-library/react' +import { apiClientV2 } from '@/api/client' +import ConnectionsTab from './ConnectionsTab' +import { useAuthStore } from '@/stores/authStore' + +vi.mock('@/api/client', () => ({ + apiClientV2: { get: vi.fn(), post: vi.fn(), delete: vi.fn() }, +})) + +describe('ConnectionsTab database connections', () => { + beforeEach(() => { + vi.clearAllMocks() + useAuthStore.setState({ user: { id: 'editor', username: 'editor', email: '', role: 'editor', is_active: true, created_at: '' } }) + }) + afterEach(() => useAuthStore.setState({ user: null, token: null })) + + it('lets viewers read connections without exposing editor-only actions', async () => { + useAuthStore.setState({ user: { ...useAuthStore.getState().user!, role: 'viewer' } }) + vi.mocked(apiClientV2.get).mockResolvedValueOnce([{ id: 'conn-1', name: 'Read-only connection', kind: 'mysql', status: 'active' }]) + render() + await screen.findByText('Read-only connection') + expect(screen.queryByRole('button', { name: /新建连接|new connection/i })).toBeNull() + expect(screen.queryByRole('button', { name: /发现数据表|discover tables/i })).toBeNull() + expect(screen.queryByRole('button', { name: /删除|delete/i })).toBeNull() + }) + + it('tests a database config before saving it', async () => { + vi.mocked(apiClientV2.get).mockResolvedValueOnce([]) + vi.mocked(apiClientV2.post) + .mockResolvedValueOnce({ success: true }) + .mockResolvedValueOnce({ id: 'conn-1' }) + .mockResolvedValueOnce([]) + render() + + await waitFor(() => expect(screen.getByRole('button', { name: /新建连接|new connection/i })).toBeTruthy()) + fireEvent.click(screen.getByRole('button', { name: /新建连接|new connection/i })) + fireEvent.change(screen.getByLabelText(/连接名称|connection name/i), { target: { value: 'Runtime MySQL' } }) + fireEvent.click(screen.getByRole('button', { name: /测试连接|test connection/i })) + + await waitFor(() => expect(screen.getByText(/连接成功|connected/i)).toBeTruthy()) + expect(apiClientV2.post).toHaveBeenCalledWith('/connections/test-config', expect.objectContaining({ type: 'mysql' })) + }) + + it('discovers tables for a saved database connection', async () => { + vi.mocked(apiClientV2.get) + .mockResolvedValueOnce([{ id: 'conn-1', name: 'Runtime MySQL', kind: 'mysql', status: 'active' }]) + .mockResolvedValueOnce({ resources: ['codex_pipeline_probe'] }) + render() + + await waitFor(() => expect(screen.getByText('Runtime MySQL')).toBeTruthy()) + fireEvent.click(screen.getByRole('button', { name: /发现数据表|discover tables/i })) + await waitFor(() => expect(screen.getByText('codex_pipeline_probe')).toBeTruthy()) + expect(apiClientV2.get).toHaveBeenCalledWith('/connections/conn-1/resources') + expect(screen.queryByRole('button', { name: /同步|sync/i })).toBeNull() + expect(screen.getByText(/Pipeline Builder/)).toBeTruthy() + }) + + it('shows a resource discovery error instead of an empty table result', async () => { + vi.mocked(apiClientV2.get) + .mockResolvedValueOnce([{ id: 'conn-1', name: 'Runtime MySQL', kind: 'mysql', status: 'active' }]) + .mockRejectedValueOnce(new Error('database unavailable')) + render() + + await waitFor(() => expect(screen.getByText('Runtime MySQL')).toBeTruthy()) + fireEvent.click(screen.getByRole('button', { name: /发现数据表|discover tables/i })) + await waitFor(() => expect(screen.getByText('database unavailable')).toBeTruthy()) + }) +}) diff --git a/frontend/src/pages/pipelines/connections/ConnectionsTab.tsx b/frontend/src/pages/pipelines/connections/ConnectionsTab.tsx index 6250a2d3..4b8b5b09 100644 --- a/frontend/src/pages/pipelines/connections/ConnectionsTab.tsx +++ b/frontend/src/pages/pipelines/connections/ConnectionsTab.tsx @@ -1,8 +1,9 @@ import { useState, useEffect, useCallback } from 'react' import { useTranslation } from 'react-i18next' import { useDropzone } from 'react-dropzone' -import { Plus, Database, FileUp, Globe, X, Loader2, RefreshCw } from 'lucide-react' +import { Plus, Database, FileUp, Globe, X, Loader2, RefreshCw, CheckCircle, XCircle } from 'lucide-react' import { apiClientV2 } from '@/api/client' +import { useAuthStore } from '@/stores/authStore' interface Connection { id: string @@ -106,12 +107,21 @@ function FileUploadZone({ files, onFilesChange }: { files: File[]; onFilesChange export default function ConnectionsTab() { const { t } = useTranslation() + const role = useAuthStore(s => s.user?.role) + const canEdit = role === 'editor' || role === 'admin' const [connections, setConnections] = useState([]) const [loading, setLoading] = useState(true) const [showForm, setShowForm] = useState(false) const [saving, setSaving] = useState(false) const [formError, setFormError] = useState('') const [syncing, setSyncing] = useState(null) + const [testing, setTesting] = useState(false) + const [testStatus, setTestStatus] = useState<'idle' | 'success' | 'failed'>('idle') + const [testMessage, setTestMessage] = useState('') + const [expandedConnection, setExpandedConnection] = useState(null) + const [resources, setResources] = useState>({}) + const [resourceErrors, setResourceErrors] = useState>({}) + const [resourceLoading, setResourceLoading] = useState(null) const [formName, setFormName] = useState('') const [formKind, setFormKind] = useState('mysql') @@ -140,9 +150,40 @@ export default function ConnectionsTab() { setFormSyncMode('snapshot') setFormFiles([]) setFormError('') + setTestStatus('idle') + setTestMessage('') + } + + const handleTest = async () => { + if (!canEdit) return + if (formKind === 'file') { + setTestStatus(formFiles.length > 0 ? 'success' : 'failed') + setTestMessage(formFiles.length > 0 ? t('connectionsTab.test_success') : t('connectionsTab.files_required_error')) + return + } + setTesting(true) + setTestStatus('idle') + setTestMessage('') + try { + const result = await apiClientV2.post('/connections/test-config', { type: formKind, config: formConfig }) as { success?: boolean; detail?: string } + if (result?.success) { + setTestStatus('success') + setTestMessage(t('connectionsTab.test_success')) + } else { + setTestStatus('failed') + setTestMessage(result?.detail || t('connectionsTab.test_failed')) + } + } catch (e: unknown) { + const err = e as { detail?: string; message?: string } | null | undefined + setTestStatus('failed') + setTestMessage(err?.detail || err?.message || t('connectionsTab.test_failed')) + } finally { + setTesting(false) + } } const handleSave = async () => { + if (!canEdit) return if (!formName.trim()) { setFormError(t('connectionsTab.name_required_error')); return } if (formKind === 'file' && formFiles.length === 0) { setFormError(t('connectionsTab.files_required_error')); return } setSaving(true) @@ -178,6 +219,7 @@ export default function ConnectionsTab() { } const handleSync = async (id: string) => { + if (!canEdit) return setSyncing(id) try { await apiClientV2.post(`/connections/${id}/sync`, {}) @@ -190,11 +232,37 @@ export default function ConnectionsTab() { } const handleDelete = async (id: string) => { + if (!canEdit) return if (!window.confirm(t('connectionsTab.confirm_delete'))) return await apiClientV2.delete(`/connections/${id}`) loadConnections() } + const discoverResources = async (id: string) => { + if (!canEdit) return + if (expandedConnection === id && resources[id]) { + setExpandedConnection(null) + return + } + setResourceLoading(id) + setResourceErrors(previous => ({ ...previous, [id]: '' })) + try { + const result = await apiClientV2.get<{ resources?: string[] }>(`/connections/${id}/resources`) + const list = Array.isArray(result) ? result as unknown as string[] : (result?.resources || []) + setResources(previous => ({ ...previous, [id]: list })) + setExpandedConnection(id) + } catch (e: unknown) { + const err = e as { detail?: string; message?: string } | null | undefined + setResourceErrors(previous => ({ + ...previous, + [id]: err?.detail || err?.message || t('connectionsTab.resources_failed'), + })) + setExpandedConnection(id) + } finally { + setResourceLoading(null) + } + } + if (loading) return
{t('common.loading')}
return ( @@ -204,15 +272,15 @@ export default function ConnectionsTab() {

{t('connectionsTab.title')}

{t('connectionsTab.subtitle')}

- + } - {showForm && ( + {canEdit && showForm && (

{t('connectionsTab.new_connection')}

@@ -222,8 +290,9 @@ export default function ConnectionsTab() {
- + setFormName(e.target.value)} placeholder={t('connectionsTab.name_ph')} @@ -287,11 +356,22 @@ export default function ConnectionsTab() {
{formError &&

{formError}

} + {testStatus !== 'idle' &&

+ {testStatus === 'success' ? : } {testMessage} +

}
+ + )} + {canEdit && (isSqlConnection ? ( + + {t('connectionsTab.pipeline_sync_hint')} + + ) : ( + + ))} + {canEdit && }
- - {statusLabel} - - - + {canEdit && expandedConnection === c.id && ( +
+ {resourceErrors[c.id] ? ( +

{resourceErrors[c.id]}

+ ) : resources[c.id]?.length ? ( +
+ {resources[c.id].map(resource => {resource})} +
+ ) :

{t('connectionsTab.no_tables')}

} +
+ )}
) })} diff --git a/frontend/src/pages/pipelines/curated/CuratedTab.test.tsx b/frontend/src/pages/pipelines/curated/CuratedTab.test.tsx new file mode 100644 index 00000000..dddd1945 --- /dev/null +++ b/frontend/src/pages/pipelines/curated/CuratedTab.test.tsx @@ -0,0 +1,19 @@ +import '@/i18n' +import { afterEach, expect, it, vi } from 'vitest' +import { render, screen } from '@testing-library/react' +import { apiClientV2 } from '@/api/client' +import { useAuthStore } from '@/stores/authStore' +import CuratedTab from './CuratedTab' + +afterEach(() => { vi.restoreAllMocks(); useAuthStore.setState({ user: null, token: null }) }) + +it.each(['viewer', 'editor', 'admin'] as const)('shows review controls only to admin (current: %s)', async role => { + useAuthStore.setState({ user: { id: role, username: role, email: '', role, is_active: true, created_at: '' } }) + vi.spyOn(apiClientV2, 'get').mockResolvedValue([{ id: 'curated-1', name: 'Review data', status: 'pending_review', kind: 'structured' }]) + render() + await screen.findByText('Review data') + const approve = screen.queryByRole('button', { name: /批准|通过|approve/i }) + const reject = screen.queryByRole('button', { name: /拒绝|reject/i }) + if (role === 'admin') { expect(approve).toBeTruthy(); expect(reject).toBeTruthy() } + else { expect(approve).toBeNull(); expect(reject).toBeNull() } +}) diff --git a/frontend/src/pages/pipelines/curated/CuratedTab.tsx b/frontend/src/pages/pipelines/curated/CuratedTab.tsx index ac7a4bb7..7024b8cc 100644 --- a/frontend/src/pages/pipelines/curated/CuratedTab.tsx +++ b/frontend/src/pages/pipelines/curated/CuratedTab.tsx @@ -5,6 +5,7 @@ import { Table2, BarChart3, Database } from 'lucide-react' import { apiClientV2 } from '@/api/client' +import { useAuthStore } from '@/stores/authStore' interface CuratedDataset { id: string @@ -49,6 +50,7 @@ const scoreColor = (s: number) => export default function CuratedTab() { const { t } = useTranslation() + const canReview = useAuthStore(s => s.user?.role === 'admin') const [datasets, setDatasets] = useState([]) const [loading, setLoading] = useState(true) const [expanded, setExpanded] = useState(null) @@ -103,6 +105,7 @@ export default function CuratedTab() { } const handleReview = async (id: string, action: 'approve' | 'reject') => { + if (!canReview) return setReviewing(id) try { await apiClientV2.post(`/curated/${id}/review?action=${action}`) @@ -177,7 +180,7 @@ export default function CuratedTab() {
- {isPending && ( + {canReview && isPending && ( <>