diff --git a/.env.example b/.env.example
index 4035784b..95db94b5 100644
--- a/.env.example
+++ b/.env.example
@@ -34,6 +34,6 @@ MINIO_SECRET_KEY=minioadmin
MINIO_USE_SSL=false
# v2 — ChromaDB
+# Docker network uses port 8000; host-side clients use localhost:8001.
CHROMA_HOST=chromadb
-CHROMA_PORT=8001
-
+CHROMA_PORT=8000
diff --git a/README.md b/README.md
index f0a289ce..01d5857a 100644
--- a/README.md
+++ b/README.md
@@ -90,7 +90,7 @@ In Ontexus, every ontology is made of these building blocks:
| Layer | Technology |
|---|---|
-| Frontend | React 18, TypeScript, Vite, Tailwind CSS, Cytoscape.js |
+| Frontend | React 19, TypeScript, Vite, Tailwind CSS, Cytoscape.js |
| Backend | FastAPI, SQLAlchemy, Alembic |
| Metadata DB | SQLite (dev) / PostgreSQL (prod) |
| Object storage | MinIO (optional, local-file fallback) |
diff --git a/backend/app/routers/actions.py b/backend/app/routers/actions.py
index a923fc67..cf3bb8fa 100644
--- a/backend/app/routers/actions.py
+++ b/backend/app/routers/actions.py
@@ -2,6 +2,7 @@
from sqlalchemy.orm import Session
from app.deps import get_db, get_current_user, require_editor
from app.models.user import User
+from app.services.ontology_access import GrantDenied, require_project_grant
from app.services.publication.working_copy import OntologyWorkingCopyService
from app.services.publication.legacy_sync import delete_action_mirror, sync_action_to_v2
from app.models.action import Action
@@ -10,13 +11,22 @@
router = APIRouter()
+
+def _grant(db: Session, actor: User, ontology_id: str, capability: str) -> dict:
+ try:
+ return require_project_grant(db, actor, ontology_id, capability)
+ except GrantDenied as exc:
+ raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc
+
@router.get("")
-def list_actions(ontology_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)):
+def list_actions(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+ _grant(db, current_user, ontology_id, "read")
items = db.query(Action).filter(Action.ontology_id == ontology_id).all()
return {"data": [ActionOut.model_validate(a).model_dump() for a in items]}
@router.post("", status_code=201)
def create_action(ontology_id: str, body: ActionCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
+ _grant(db, current_user, ontology_id, "edit")
def _write():
data = {k: v for k, v in body.model_dump().items() if v is not None}
a = Action(id=str(uuid.uuid4()), ontology_id=ontology_id, **data)
@@ -29,7 +39,8 @@ def _write():
return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="action.create", callback=_write)
@router.get("/{action_id}")
-def get_action(ontology_id: str, action_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)):
+def get_action(ontology_id: str, action_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+ _grant(db, current_user, ontology_id, "read")
a = db.query(Action).filter(Action.id == action_id, Action.ontology_id == ontology_id).first()
if not a:
raise HTTPException(404, "Not found")
@@ -37,6 +48,7 @@ def get_action(ontology_id: str, action_id: str, db: Session = Depends(get_db),
@router.put("/{action_id}")
def update_action(ontology_id: str, action_id: str, body: ActionUpdate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
+ _grant(db, current_user, ontology_id, "edit")
a = db.query(Action).filter(Action.id == action_id, Action.ontology_id == ontology_id).first()
if not a:
raise HTTPException(404, "Not found")
@@ -51,6 +63,7 @@ def _write():
@router.delete("/{action_id}", status_code=204)
def delete_action(ontology_id: str, action_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
+ _grant(db, current_user, ontology_id, "edit")
a = db.query(Action).filter(Action.id == action_id, Action.ontology_id == ontology_id).first()
if not a:
raise HTTPException(404, "Not found")
@@ -62,6 +75,7 @@ def _write():
@router.post("/{action_id}/toggle")
def toggle_action(ontology_id: str, action_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
+ _grant(db, current_user, ontology_id, "edit")
a = db.query(Action).filter(Action.id == action_id, Action.ontology_id == ontology_id).first()
if not a:
raise HTTPException(404, "Not found")
@@ -74,6 +88,7 @@ def _write():
@router.post("/publish")
-def publish_actions(ontology_id: str, db: Session = Depends(get_db), _=Depends(require_editor)):
+def publish_actions(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
"""Direct tool publication is closed; use the Ontology compiler (P1C)."""
+ _grant(db, current_user, ontology_id, "edit")
raise HTTPException(status_code=403, detail="PUBLICATION_NOT_ENABLED")
diff --git a/backend/app/routers/audit.py b/backend/app/routers/audit.py
index 88246768..5ee5c3ac 100644
--- a/backend/app/routers/audit.py
+++ b/backend/app/routers/audit.py
@@ -5,13 +5,19 @@
from app.models.ontology import OntologyProject
from app.schemas.audit import AuditRequest, AuditTaskOut
from app.services.publication.extraction_model_port import default_extraction_model_port
+from app.models.user import User
+from app.services.ontology_access import GrantDenied, require_project_grant
import uuid
router = APIRouter()
@router.post("")
-def start_audit(ontology_id: str, body: AuditRequest, db: Session = Depends(get_db), _=Depends(require_editor)):
+def start_audit(ontology_id: str, body: AuditRequest, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
+ try:
+ require_project_grant(db, current_user, ontology_id, "edit")
+ except GrantDenied as exc:
+ raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc
project = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first()
if not project:
raise HTTPException(404, "Ontology not found")
@@ -50,7 +56,11 @@ def run_sync():
@router.get("/status")
-def get_audit_status(ontology_id: str, task_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)):
+def get_audit_status(ontology_id: str, task_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+ try:
+ require_project_grant(db, current_user, ontology_id, "read")
+ except GrantDenied as exc:
+ raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc
task = db.query(AuditTask).filter(AuditTask.id == task_id, AuditTask.ontology_id == ontology_id).first()
if not task:
raise HTTPException(404, "Audit task not found")
diff --git a/backend/app/routers/entities.py b/backend/app/routers/entities.py
index d8651313..d3fa0358 100644
--- a/backend/app/routers/entities.py
+++ b/backend/app/routers/entities.py
@@ -3,19 +3,29 @@
from app.deps import get_db, get_current_user, require_admin, require_editor
from app.models.entity import Entity
from app.models.user import User
+from app.services.ontology_access import GrantDenied, require_project_grant
from app.services.publication.working_copy import OntologyWorkingCopyService
from app.schemas.entity import EntityCreate, EntityUpdate, EntityOut
import uuid
router = APIRouter()
+
+def _grant(db: Session, actor: User, ontology_id: str, capability: str) -> dict:
+ try:
+ return require_project_grant(db, actor, ontology_id, capability)
+ except GrantDenied as exc:
+ raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc
+
@router.get("")
-def list_entities(ontology_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)):
+def list_entities(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+ _grant(db, current_user, ontology_id, "read")
items = db.query(Entity).filter(Entity.ontology_id == ontology_id).all()
return {"data": [EntityOut.model_validate(e).model_dump() for e in items]}
@router.post("", status_code=201)
def create_entity(ontology_id: str, body: EntityCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
+ _grant(db, current_user, ontology_id, "edit")
def _write():
data = {k: v for k, v in body.model_dump().items() if v is not None}
e = Entity(id=str(uuid.uuid4()), ontology_id=ontology_id, **data)
@@ -24,7 +34,8 @@ def _write():
return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="entity.create", callback=_write)
@router.get("/{entity_id}")
-def get_entity(ontology_id: str, entity_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)):
+def get_entity(ontology_id: str, entity_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+ _grant(db, current_user, ontology_id, "read")
e = db.query(Entity).filter(Entity.id == entity_id, Entity.ontology_id == ontology_id).first()
if not e:
raise HTTPException(404, "Not found")
@@ -32,6 +43,7 @@ def get_entity(ontology_id: str, entity_id: str, db: Session = Depends(get_db),
@router.put("/{entity_id}")
def update_entity(ontology_id: str, entity_id: str, body: EntityUpdate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
+ _grant(db, current_user, ontology_id, "edit")
e = db.query(Entity).filter(Entity.id == entity_id, Entity.ontology_id == ontology_id).first()
if not e:
raise HTTPException(404, "Not found")
@@ -44,6 +56,7 @@ def _write():
@router.delete("/{entity_id}", status_code=204)
def delete_entity(ontology_id: str, entity_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_admin)):
+ _grant(db, current_user, ontology_id, "edit")
e = db.query(Entity).filter(Entity.id == entity_id, Entity.ontology_id == ontology_id).first()
if not e:
raise HTTPException(404, "Not found")
@@ -56,9 +69,10 @@ def list_entity_instances(
ontology_id: str,
entity_id: str,
db: Session = Depends(get_db),
- _=Depends(get_current_user),
+ current_user: User = Depends(get_current_user),
):
"""概念实体下挂的行级实例数据(Pipeline Mapping 与简易 LLM 提取共用同一张表)"""
+ _grant(db, current_user, ontology_id, "read")
from app.models.entity_instance import EntityInstance
entity = db.query(Entity).filter(
diff --git a/backend/app/routers/export.py b/backend/app/routers/export.py
index 8604fdc2..ae6078e0 100644
--- a/backend/app/routers/export.py
+++ b/backend/app/routers/export.py
@@ -3,6 +3,7 @@
from sqlalchemy.orm import Session
from app.deps import get_db, get_current_user
from app.services import export_service
+from app.services.ontology_access import GrantDenied, require_project_grant
router = APIRouter()
@@ -21,8 +22,12 @@ def export_ontology(
ontology_id: str,
format: str = Query("json", pattern="^(json|yaml|csv|ttl|html|cypher|tugraph)$"),
db: Session = Depends(get_db),
- _=Depends(get_current_user)
+ current_user=Depends(get_current_user)
):
+ try:
+ require_project_grant(db, current_user, ontology_id, "read")
+ except GrantDenied as exc:
+ raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc
if format not in FORMAT_MAP:
raise HTTPException(400, f"Unsupported format: {format}")
content_type, ext, fn = FORMAT_MAP[format]
diff --git a/backend/app/routers/extraction.py b/backend/app/routers/extraction.py
index 81a27ec2..6d9828d6 100644
--- a/backend/app/routers/extraction.py
+++ b/backend/app/routers/extraction.py
@@ -9,6 +9,7 @@
from app.models.ontology import OntologyProject
from app.services.document_service import combine_converted_files
from app.schemas.extraction import ExtractionRequest, ExtractionTaskOut
+from app.services.ontology_access import GrantDenied, require_project_grant
import uuid
logger = logging.getLogger(__name__)
@@ -16,7 +17,11 @@
router = APIRouter()
@router.post("")
-def start_extraction(ontology_id: str, body: ExtractionRequest, db: Session = Depends(get_db), _=Depends(require_editor)):
+def start_extraction(ontology_id: str, body: ExtractionRequest, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
+ try:
+ require_project_grant(db, current_user, ontology_id, "edit")
+ except GrantDenied as exc:
+ raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc
project = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first()
if not project:
raise HTTPException(404, "Ontology not found")
@@ -67,7 +72,11 @@ def run_sync():
return {"data": {"task_id": task.id}, "message": "Extraction queued"}
@router.get("/status")
-def get_extraction_status(ontology_id: str, task_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)):
+def get_extraction_status(ontology_id: str, task_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+ try:
+ require_project_grant(db, current_user, ontology_id, "read")
+ except GrantDenied as exc:
+ raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc
task = db.query(ExtractionTask).filter(ExtractionTask.id == task_id, ExtractionTask.ontology_id == ontology_id).first()
if not task:
raise HTTPException(404, "Task not found")
diff --git a/backend/app/routers/files.py b/backend/app/routers/files.py
index 926e1d14..52f7480c 100644
--- a/backend/app/routers/files.py
+++ b/backend/app/routers/files.py
@@ -1,5 +1,6 @@
import os
import uuid
+import logging
from fastapi import APIRouter, Depends, HTTPException, UploadFile, File
from sqlalchemy.orm import Session
from app.deps import get_db, get_current_user, require_admin, require_editor
@@ -7,9 +8,11 @@
from app.models.ontology import OntologyProject
from app.schemas.file import FileOut
from app.services.document_service import convert_document, is_usable_converted_text
+from app.services.ontology_access import GrantDenied, require_project_grant
from app.config import settings
router = APIRouter()
+logger = logging.getLogger(__name__)
ALLOWED_TYPES = {
"application/pdf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
@@ -39,8 +42,16 @@ def _file_out(f: UploadedFile) -> dict:
).model_dump()
+def _grant(db: Session, actor, ontology_id: str, capability: str) -> dict:
+ try:
+ return require_project_grant(db, actor, ontology_id, capability)
+ except GrantDenied as exc:
+ raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc
+
+
@router.get("")
-def list_files(ontology_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)):
+def list_files(ontology_id: str, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
+ _grant(db, current_user, ontology_id, "read")
files = db.query(UploadedFile).filter(UploadedFile.ontology_id == ontology_id).all()
return {"data": [_file_out(f) for f in files]}
@@ -49,11 +60,12 @@ async def upload_file(
ontology_id: str,
file: UploadFile = File(...),
db: Session = Depends(get_db),
- _=Depends(require_editor)
+ current_user=Depends(require_editor),
):
+ _grant(db, current_user, ontology_id, "edit")
project = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first()
if not project:
- raise HTTPException(404, "Ontology not found")
+ raise HTTPException(404, detail="ONTOLOGY_NOT_FOUND")
ext_name = (file.filename or "").rsplit(".", 1)[-1].lower()
allowed = {e.strip() for e in settings.allowed_upload_extensions.split(",") if e.strip()}
@@ -67,36 +79,63 @@ async def upload_file(
ext = os.path.splitext(file.filename)[1]
save_path = os.path.join(upload_dir, f"{file_id}{ext}")
- content = await file.read()
- if len(content) > settings.max_upload_mb * 1024 * 1024:
+ max_bytes = settings.max_upload_mb * 1024 * 1024
+ declared_size = getattr(file, "size", None)
+ if declared_size is not None and declared_size > max_bytes:
raise HTTPException(413, f"文件超过大小限制 {settings.max_upload_mb}MB")
- with open(save_path, "wb") as f:
- f.write(content)
-
- mime = file.content_type or "application/octet-stream"
- conversion = convert_document(save_path, mime)
- if not conversion.ok:
- if os.path.exists(save_path):
- os.remove(save_path)
- raise HTTPException(422, conversion.error or "文件转换失败")
-
- db_file = UploadedFile(
- id=file_id,
- ontology_id=ontology_id,
- filename=file.filename,
- file_path=save_path,
- file_size=len(content),
- mime_type=mime,
- converted_md=conversion.content,
- )
- db.add(db_file); db.commit(); db.refresh(db_file)
- return {"data": _file_out(db_file)}
+
+ committed = False
+ try:
+ total_bytes = 0
+ with open(save_path, "wb") as saved:
+ while True:
+ chunk = await file.read(1024 * 1024)
+ if not chunk:
+ break
+ total_bytes += len(chunk)
+ if total_bytes > max_bytes:
+ raise HTTPException(413, f"文件超过大小限制 {settings.max_upload_mb}MB")
+ saved.write(chunk)
+
+ mime = file.content_type or "application/octet-stream"
+ conversion = convert_document(save_path, mime)
+ if not conversion.ok:
+ raise HTTPException(422, conversion.error or "文件转换失败")
+
+ db_file = UploadedFile(
+ id=file_id,
+ ontology_id=ontology_id,
+ filename=file.filename,
+ file_path=save_path,
+ file_size=total_bytes,
+ mime_type=mime,
+ converted_md=conversion.content,
+ )
+ db.add(db_file)
+ try:
+ db.commit()
+ except Exception:
+ db.rollback()
+ raise
+ committed = True
+ db.refresh(db_file)
+ return {"data": _file_out(db_file)}
+ except Exception:
+ if not committed and os.path.exists(save_path):
+ try:
+ os.remove(save_path)
+ except OSError:
+ logger.warning("failed to clean up rejected upload %s", save_path, exc_info=True)
+ raise
@router.delete("/{file_id}", status_code=204)
-def delete_file(ontology_id: str, file_id: str, db: Session = Depends(get_db), _=Depends(require_admin)):
+def delete_file(ontology_id: str, file_id: str, db: Session = Depends(get_db), current_user=Depends(require_admin)):
+ _grant(db, current_user, ontology_id, "edit")
f = db.query(UploadedFile).filter(UploadedFile.id == file_id, UploadedFile.ontology_id == ontology_id).first()
if not f:
raise HTTPException(404, "File not found")
- if os.path.exists(f.file_path):
- os.remove(f.file_path)
- db.delete(f); db.commit()
+ file_path = f.file_path
+ db.delete(f)
+ db.commit()
+ if os.path.exists(file_path):
+ os.remove(file_path)
diff --git a/backend/app/routers/graph.py b/backend/app/routers/graph.py
index ea6bca00..61aeee57 100644
--- a/backend/app/routers/graph.py
+++ b/backend/app/routers/graph.py
@@ -5,13 +5,22 @@
from app.models.relation import Relation
from app.models.ontology import OntologyProject
from app.models.user import User
+from app.services.ontology_access import GrantDenied, require_project_grant
from app.services.publication.working_copy import OntologyWorkingCopyService
from app.services.relation_dedup import dedupe_relation_rows
router = APIRouter()
+
+def _grant(db: Session, actor: User, ontology_id: str, capability: str) -> dict:
+ try:
+ return require_project_grant(db, actor, ontology_id, capability)
+ except GrantDenied as exc:
+ raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc
+
@router.get("")
-def get_graph(ontology_id: str, limit: int = 300, db: Session = Depends(get_db), _=Depends(get_current_user)):
+def get_graph(ontology_id: str, limit: int = 300, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+ _grant(db, current_user, ontology_id, "read")
project = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first()
if not project:
raise HTTPException(404, "Ontology not found")
@@ -138,12 +147,26 @@ def create_relation(
):
from app.models.relation import Relation
import uuid
+ _grant(db, current_user, ontology_id, "edit")
+ source_entity = body.get("source_entity")
+ target_entity = body.get("target_entity")
+ if not source_entity or not target_entity:
+ raise HTTPException(status_code=422, detail="source_entity and target_entity are required")
+ source = db.query(Entity).filter(
+ Entity.id == source_entity, Entity.ontology_id == ontology_id
+ ).first()
+ target = db.query(Entity).filter(
+ Entity.id == target_entity, Entity.ontology_id == ontology_id
+ ).first()
+ if source is None or target is None:
+ raise HTTPException(status_code=422, detail="RELATION_ENTITIES_MUST_BELONG_TO_ONTOLOGY")
+
def _write():
relation = Relation(
id=str(uuid.uuid4()),
ontology_id=ontology_id,
- source_entity=body["source_entity"],
- target_entity=body["target_entity"],
+ source_entity=source_entity,
+ target_entity=target_entity,
type=body.get("type", "关联"),
properties=body.get("properties", {}),
confidence=body.get("confidence", 1.0),
@@ -154,6 +177,7 @@ def _write():
@router.delete("/relations/{relation_id}", status_code=204)
def delete_relation(ontology_id: str, relation_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_admin)):
+ _grant(db, current_user, ontology_id, "edit")
r = db.query(Relation).filter(Relation.id == relation_id, Relation.ontology_id == ontology_id).first()
if not r:
raise HTTPException(404, "Not found")
diff --git a/backend/app/routers/logic.py b/backend/app/routers/logic.py
index f2e9b51b..8b3e867b 100644
--- a/backend/app/routers/logic.py
+++ b/backend/app/routers/logic.py
@@ -2,6 +2,7 @@
from sqlalchemy.orm import Session
from app.deps import get_db, get_current_user, require_editor
from app.models.user import User
+from app.services.ontology_access import GrantDenied, require_project_grant
from app.services.publication.working_copy import OntologyWorkingCopyService
from app.services.publication.legacy_sync import delete_logic_rule_mirror, sync_logic_rule_to_v2
from app.models.logic import LogicRule
@@ -10,13 +11,22 @@
router = APIRouter()
+
+def _grant(db: Session, actor: User, ontology_id: str, capability: str) -> dict:
+ try:
+ return require_project_grant(db, actor, ontology_id, capability)
+ except GrantDenied as exc:
+ raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc
+
@router.get("")
-def list_logic(ontology_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)):
+def list_logic(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+ _grant(db, current_user, ontology_id, "read")
items = db.query(LogicRule).filter(LogicRule.ontology_id == ontology_id).all()
return {"data": [LogicRuleOut.model_validate(r).model_dump() for r in items]}
@router.post("", status_code=201)
def create_logic(ontology_id: str, body: LogicRuleCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
+ _grant(db, current_user, ontology_id, "edit")
def _write():
data = {k: v for k, v in body.model_dump().items() if v is not None}
r = LogicRule(id=str(uuid.uuid4()), ontology_id=ontology_id, **data)
@@ -29,7 +39,8 @@ def _write():
return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="logic.create", callback=_write)
@router.get("/{logic_id}")
-def get_logic(ontology_id: str, logic_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)):
+def get_logic(ontology_id: str, logic_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+ _grant(db, current_user, ontology_id, "read")
r = db.query(LogicRule).filter(LogicRule.id == logic_id, LogicRule.ontology_id == ontology_id).first()
if not r:
raise HTTPException(404, "Not found")
@@ -37,6 +48,7 @@ def get_logic(ontology_id: str, logic_id: str, db: Session = Depends(get_db), _=
@router.put("/{logic_id}")
def update_logic(ontology_id: str, logic_id: str, body: LogicRuleUpdate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
+ _grant(db, current_user, ontology_id, "edit")
r = db.query(LogicRule).filter(LogicRule.id == logic_id, LogicRule.ontology_id == ontology_id).first()
if not r:
raise HTTPException(404, "Not found")
@@ -51,6 +63,7 @@ def _write():
@router.delete("/{logic_id}", status_code=204)
def delete_logic(ontology_id: str, logic_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
+ _grant(db, current_user, ontology_id, "edit")
r = db.query(LogicRule).filter(LogicRule.id == logic_id, LogicRule.ontology_id == ontology_id).first()
if not r:
raise HTTPException(404, "Not found")
@@ -63,6 +76,7 @@ def _write():
@router.post("/{logic_id}/toggle")
def toggle_logic(ontology_id: str, logic_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
"""Human Review: 启用/禁用规则"""
+ _grant(db, current_user, ontology_id, "edit")
r = db.query(LogicRule).filter(LogicRule.id == logic_id, LogicRule.ontology_id == ontology_id).first()
if not r:
raise HTTPException(404, "Not found")
@@ -75,6 +89,7 @@ def _write():
@router.post("/publish")
-def publish_logic_rules(ontology_id: str, db: Session = Depends(get_db), _=Depends(require_editor)):
+def publish_logic_rules(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
"""Human Review: 发布所有草稿规则 — direct tool publication is closed; use the Ontology compiler (P1C)."""
+ _grant(db, current_user, ontology_id, "edit")
raise HTTPException(status_code=403, detail="PUBLICATION_NOT_ENABLED")
diff --git a/backend/app/routers/models.py b/backend/app/routers/models.py
index 58227054..9b68eac2 100644
--- a/backend/app/routers/models.py
+++ b/backend/app/routers/models.py
@@ -39,6 +39,7 @@
remediate_blocked_identity,
rotate_credential,
select_active_version,
+ versioning_schema_present,
)
router = APIRouter()
@@ -325,14 +326,32 @@ def delete_model(model_id: str, db: Session = Depends(get_db), _=Depends(require
c = db.query(ModelConfig).filter(ModelConfig.id == model_id).first()
if not c:
raise HTTPException(404, "Not found")
+ # Block only on a REAL dependent — an Agent version pinned to one of this
+ # model's versions, or a prompt-generation provenance row — not merely on
+ # the model's own version-1 row that create_model() always bootstraps.
+ # Checking "any model_config_versions row exists" made every model saved
+ # through the normal create flow permanently undeletable.
referenced = db.execute(text(
- "SELECT count(*) FROM model_config_versions WHERE model_config_id = :id"
+ "SELECT count(*) FROM model_config_versions v WHERE v.model_config_id = :id AND ("
+ "EXISTS (SELECT 1 FROM agent_versions av WHERE av.default_model_config_version_id = v.id) "
+ "OR EXISTS (SELECT 1 FROM prompt_generations pg WHERE pg.model_config_version_id = v.id))"
), {"id": model_id}).scalar_one()
if referenced > 0:
raise HTTPException(409, detail="MODEL_REFERENCED")
db.query(ExtractionTask).filter(ExtractionTask.model_id == model_id).update(
{ExtractionTask.model_id: None}, synchronize_session=False
)
+ # model_config_versions/model_credentials/model_migration_findings all
+ # RESTRICT-reference model_configs.id, so they must be cleared first —
+ # and model_configs.active_version_id RESTRICT-references the version
+ # row itself (the reverse direction), so that pointer must be nulled
+ # before the version row can go. The pointer is migration-only; the
+ # legacy metadata.create_all schema has no such column.
+ if versioning_schema_present(db):
+ db.execute(text("UPDATE model_configs SET active_version_id = NULL WHERE id = :id"), {"id": model_id})
+ db.execute(text("DELETE FROM model_credentials WHERE model_config_id = :id"), {"id": model_id})
+ db.execute(text("DELETE FROM model_migration_findings WHERE model_config_id = :id"), {"id": model_id})
+ db.execute(text("DELETE FROM model_config_versions WHERE model_config_id = :id"), {"id": model_id})
db.delete(c)
db.commit()
diff --git a/backend/app/routers/ontologies.py b/backend/app/routers/ontologies.py
index 143d9b38..62ef6360 100644
--- a/backend/app/routers/ontologies.py
+++ b/backend/app/routers/ontologies.py
@@ -1,3 +1,5 @@
+import logging
+
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy.orm import Session
from sqlalchemy import func, text
@@ -9,19 +11,34 @@
from app.models.relation import Relation
from app.models.user import User
from app.schemas.ontology import OntologyCreate, OntologyOut, OntologyListItem, OntologyUpdate
-from app.services.ontology_access import creator_grant
+from app.services.ontology_access import (
+ GrantDenied,
+ creator_grant,
+ discoverable_ontology_ids,
+ require_project_grant,
+)
from app.services.publication.working_copy import OntologyWorkingCopyService
import uuid
+logger = logging.getLogger(__name__)
router = APIRouter()
+
+def _grant(db: Session, actor: User, ontology_id: str, capability: str) -> dict:
+ try:
+ return require_project_grant(db, actor, ontology_id, capability)
+ except GrantDenied as exc:
+ raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc
+
@router.get("")
def list_ontologies(
name: Optional[str] = None,
page: int = 1, page_size: int = 20,
- db: Session = Depends(get_db), _=Depends(get_current_user)
+ db: Session = Depends(get_db), current_user: User = Depends(get_current_user)
):
- q = db.query(OntologyProject)
+ q = db.query(OntologyProject).filter(
+ OntologyProject.id.in_(discoverable_ontology_ids(db, current_user))
+ )
if name:
q = q.filter(OntologyProject.name.ilike(f"%{name}%"))
total = q.count()
@@ -41,7 +58,8 @@ def create_ontology(body: OntologyCreate, db: Session = Depends(get_db), current
raise HTTPException(status_code=409, detail={"error": "DUPLICATE_NAME", "message": f"Ontology 名称「{body.name}」已存在", "existing_id": existing.id})
project = OntologyProject(id=str(uuid.uuid4()), name=body.name, domain=body.domain,
description=body.description, build_mode=body.build_mode or "simple_llm",
- created_by=current_user.id)
+ created_by=current_user.id,
+ security_domain_id=current_user.security_domain_id)
db.add(project); db.flush()
# atomically insert the exact creator grant (P1A-ACCESS contract)
creator_grant(db, project.id, current_user)
@@ -49,17 +67,19 @@ def create_ontology(body: OntologyCreate, db: Session = Depends(get_db), current
return {"data": OntologyOut.model_validate(project).model_dump()}
@router.get("/{ontology_id}")
-def get_ontology(ontology_id: str, db: Session = Depends(get_db), _=Depends(get_current_user)):
+def get_ontology(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+ _grant(db, current_user, ontology_id, "read")
p = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first()
if not p:
- raise HTTPException(404, "Not found")
+ raise HTTPException(404, detail="ONTOLOGY_NOT_FOUND")
return {"data": OntologyOut.model_validate(p).model_dump()}
@router.put("/{ontology_id}")
def update_ontology(ontology_id: str, body: OntologyUpdate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
+ _grant(db, current_user, ontology_id, "edit")
p = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first()
if not p:
- raise HTTPException(404, "Not found")
+ raise HTTPException(404, detail="ONTOLOGY_NOT_FOUND")
def _write():
for k, v in body.model_dump(exclude_none=True).items():
setattr(p, k, v)
@@ -68,10 +88,11 @@ def _write():
return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="ontology.update", callback=_write)
@router.delete("/{ontology_id}", status_code=204)
-def delete_ontology(ontology_id: str, db: Session = Depends(get_db), _=Depends(require_admin)):
+def delete_ontology(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_admin)):
+ _grant(db, current_user, ontology_id, "edit")
p = db.query(OntologyProject).filter(OntologyProject.id == ontology_id).first()
if not p:
- raise HTTPException(404, "Not found")
+ raise HTTPException(404, detail="ONTOLOGY_NOT_FOUND")
bound_agent = db.execute(text(
"SELECT 1 FROM agent_ontology_bindings WHERE ontology_id = :id LIMIT 1"
), {"id": ontology_id}).scalar_one_or_none()
@@ -82,9 +103,14 @@ def delete_ontology(ontology_id: str, db: Session = Depends(get_db), _=Depends(r
# gone, unlike execution history (agent runs, sandbox simulations,
# business-journey records, MCP write requests) rooted through
# ontology_releases, which a delete must not silently wipe.
+ # entity_instance_relations (and entity_instances, cascade-deleted below)
+ # carry an AFTER DELETE trigger that re-emits agent_index_outbox rows —
+ # agent_index_outbox must be cleaned LAST so it sweeps up anything those
+ # triggers just recreated, or the delete below hits ONTOLOGY_HAS_DEPENDENT_DATA
+ # on an ontology that still has real instance data.
for table in (
"ontology_data_grants", "ontology_project_access_grants", "ontology_migration_findings",
- "agent_index_outbox", "entity_property_definitions", "entity_instance_relations",
+ "entity_property_definitions", "entity_instance_relations", "agent_index_outbox",
):
db.execute(text(f"DELETE FROM {table} WHERE ontology_id = :id"), {"id": ontology_id})
try:
@@ -93,3 +119,18 @@ def delete_ontology(ontology_id: str, db: Session = Depends(get_db), _=Depends(r
except IntegrityError:
db.rollback()
raise HTTPException(409, detail="ONTOLOGY_HAS_DEPENDENT_DATA")
+
+ # Best-effort external-store cleanup — never fails an already-committed
+ # delete; Neo4j/Chroma hold per-ontology graph/vector data with no other
+ # owner once the Postgres row is gone (mirrors the try/except-and-log
+ # pattern in app/tasks/extraction.py's _sync_neo4j).
+ try:
+ from app.services.v2.graph.neo4j_service import Neo4jService
+ Neo4jService().delete_by_ontology(ontology_id)
+ except Exception:
+ logger.warning("Neo4j cleanup failed for deleted ontology %s", ontology_id, exc_info=True)
+ try:
+ from app.services.v2.vector.chroma_service import ChromaService
+ ChromaService().delete_collection(f"ontology_{ontology_id}")
+ except Exception:
+ logger.warning("Chroma cleanup failed for deleted ontology %s", ontology_id, exc_info=True)
diff --git a/backend/app/routers/overview.py b/backend/app/routers/overview.py
index 18247dc6..420f4dba 100644
--- a/backend/app/routers/overview.py
+++ b/backend/app/routers/overview.py
@@ -3,23 +3,29 @@
from sqlalchemy import func
from app.deps import get_db, get_current_user
from app.models.ontology import OntologyProject
+from app.models.user import User
+from app.services.ontology_access import discoverable_ontology_ids
router = APIRouter()
-def _safe_count(db, model):
+def _safe_count(db, model, ontology_ids):
try:
- return db.query(model).count()
+ return db.query(model).filter(model.ontology_id.in_(ontology_ids)).count()
except Exception:
return 0
@router.get("/stats")
-def get_stats(db: Session = Depends(get_db), _=Depends(get_current_user)):
+def get_stats(db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
from app.models.entity import Entity
from app.models.logic import LogicRule
from app.models.action import Action
+ discoverable_ids = discoverable_ontology_ids(db, current_user)
+
# Recent ontologies
- recent = db.query(OntologyProject).order_by(OntologyProject.updated_at.desc()).limit(6).all()
+ recent = db.query(OntologyProject).filter(
+ OntologyProject.id.in_(discoverable_ids)
+ ).order_by(OntologyProject.updated_at.desc()).limit(6).all()
recent_list = []
for o in recent:
entity_count = db.query(Entity).filter(Entity.ontology_id == o.id).count()
@@ -39,6 +45,7 @@ def get_stats(db: Session = Depends(get_db), _=Depends(get_current_user)):
# Domain distribution
domain_rows = (
db.query(OntologyProject.domain, func.count(OntologyProject.id))
+ .filter(OntologyProject.id.in_(discoverable_ids))
.group_by(OntologyProject.domain)
.all()
)
@@ -47,6 +54,7 @@ def get_stats(db: Session = Depends(get_db), _=Depends(get_current_user)):
# Status breakdown
status_rows = (
db.query(OntologyProject.status, func.count(OntologyProject.id))
+ .filter(OntologyProject.id.in_(discoverable_ids))
.group_by(OntologyProject.status)
.all()
)
@@ -54,10 +62,10 @@ def get_stats(db: Session = Depends(get_db), _=Depends(get_current_user)):
return {
"data": {
- "ontology_count": _safe_count(db, OntologyProject),
- "entity_count": _safe_count(db, Entity),
- "logic_count": _safe_count(db, LogicRule),
- "action_count": _safe_count(db, Action),
+ "ontology_count": db.query(OntologyProject).filter(OntologyProject.id.in_(discoverable_ids)).count(),
+ "entity_count": _safe_count(db, Entity, discoverable_ids),
+ "logic_count": _safe_count(db, LogicRule, discoverable_ids),
+ "action_count": _safe_count(db, Action, discoverable_ids),
"recent_ontologies": recent_list,
"domain_counts": domain_counts,
"status_counts": status_counts,
diff --git a/backend/app/routers/v2/connections.py b/backend/app/routers/v2/connections.py
index 5645581c..36e30900 100644
--- a/backend/app/routers/v2/connections.py
+++ b/backend/app/routers/v2/connections.py
@@ -11,7 +11,7 @@
import json
from datetime import datetime, timezone
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, Query
from pydantic import BaseModel
from sqlalchemy.orm import Session
from typing import Optional
@@ -19,6 +19,12 @@
from app.database import SessionLocal
from app.deps import get_current_user, require_editor
from app.models.v2.connection import Connection
+from app.models.user import User
+from app.services.connection.config import (
+ ConnectionConfigError,
+ build_connector_config,
+ safe_connection_error,
+)
from app.services.connection.registry import get_connector
router = APIRouter(dependencies=[Depends(get_current_user)])
@@ -53,7 +59,7 @@ class Config:
# ── 端点 ──────────────────────────────────────────────────────
@router.post("", response_model=ConnectionResponse, status_code=201)
-def create_connection(body: ConnectionCreate, db: Session = Depends(get_db), _=Depends(require_editor)):
+def create_connection(body: ConnectionCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
"""创建连接。config 加密后存储。"""
from app.services import encryption_service
encrypted_config = {"_encrypted": encryption_service.encrypt(json.dumps(body.config))}
@@ -63,6 +69,7 @@ def create_connection(body: ConnectionCreate, db: Session = Depends(get_db), _=D
kind=body.kind,
config=encrypted_config,
status="inactive",
+ created_by=current_user.id,
)
db.add(conn)
db.commit()
@@ -94,35 +101,36 @@ def _build_db_config(raw_config: dict, db_type: str) -> dict:
connection_string,这里组装成 SQLAlchemy 可用的连接 URL。
密码中的特殊字符通过 urllib.parse.quote 编码以避免 URL 解析歧义。
"""
- from urllib.parse import quote
- host = raw_config.get("host", "localhost")
- port = raw_config.get("port", "3306" if db_type == "mysql" else "5432")
- user = raw_config.get("user", "")
- password = raw_config.get("password", "")
- database = raw_config.get("database", "")
- # 密码/用户名/库名中的特殊字符(如 @ : / # 空格等)必须 URL 编码,
- # 否则 SQLAlchemy 的 URL 解析器会将 @ 等视为 URL 结构分隔符而非密码的一部分。
- # host 不编码(IPv6 地址用 [] 括起,需原样保留)。
- scheme = "mysql+pymysql" if db_type == "mysql" else "postgresql"
- conn_str = f"{scheme}://{quote(user, safe='')}:{quote(password, safe='')}@{host}:{port}/{quote(database, safe='')}"
- config = dict(raw_config)
- config["connection_string"] = conn_str
- return config
+ return build_connector_config(db_type, raw_config)
+
+
+def _load_connection_connector(
+ conn: Connection,
+ *,
+ resource: str | None = None,
+):
+ """Build a connector from an encrypted Connection row.
+
+ Keeping this in one helper ensures API tests, pipeline execution, and
+ refresh workers all resolve the same saved database configuration.
+ """
+ config = build_connector_config(conn.kind, conn.config, resource=resource)
+ return get_connector(conn.kind, config)
@router.post("/test-config")
def test_connection_config(body: TestConfigBody, _=Depends(require_editor)):
"""测试连接配置(无需先创建 Connection,供 Builder 使用)"""
try:
- cfg = body.config
- if body.type in ("mysql", "postgresql", "postgres"):
- db_type = "postgres" if "postgres" in body.type else "mysql"
- cfg = _build_db_config(cfg, db_type)
+ cfg = build_connector_config(body.type, body.config)
connector = get_connector(body.type, cfg)
ok = connector.test_connection()
- return {"success": ok}
+ result = {"success": ok}
+ if not ok:
+ result["detail"] = getattr(connector, "last_error", None) or "connection test failed"
+ return result
except Exception as e:
- return {"success": False, "detail": str(e)}
+ return {"success": False, "detail": safe_connection_error(e)}
@router.post("/{connection_id}/test")
@@ -132,27 +140,67 @@ def test_connection(connection_id: str, db: Session = Depends(get_db), _=Depends
if not conn:
raise HTTPException(status_code=404, detail="Connection not found")
- from app.services import encryption_service
- raw = conn.config.get("_encrypted", "")
try:
- config = json.loads(encryption_service.decrypt(raw)) if raw else conn.config
- except Exception:
- config = conn.config
-
- # 与 test-config 一样,ConnectionsTab 存储的是独立字段而非 connection_string
- if conn.kind in ("mysql", "postgres"):
- config = _build_db_config(config, conn.kind)
-
- try:
- connector = get_connector(conn.kind, config)
+ connector = _load_connection_connector(conn)
ok = connector.test_connection()
conn.status = "active" if ok else "error"
db.commit()
- return {"success": ok, "status": conn.status}
+ result = {"success": ok, "status": conn.status}
+ if not ok:
+ result["detail"] = getattr(connector, "last_error", None) or "connection test failed"
+ return result
except Exception as e:
conn.status = "error"
db.commit()
- return {"success": False, "status": "error", "detail": str(e)}
+ return {"success": False, "status": "error", "detail": safe_connection_error(e)}
+
+
+@router.get("/{connection_id}/resources")
+def list_connection_resources(
+ connection_id: str,
+ db: Session = Depends(get_db),
+ _=Depends(require_editor),
+):
+ """List tables/collections exposed by a saved connection."""
+ conn = db.query(Connection).filter(Connection.id == connection_id).first()
+ if not conn:
+ raise HTTPException(404, "Connection not found")
+ try:
+ connector = _load_connection_connector(conn)
+ resources = connector.list_resources()
+ return {"connection_id": connection_id, "resources": resources}
+ except (ConnectionConfigError, ValueError) as exc:
+ raise HTTPException(422, detail=str(exc)) from exc
+ except Exception as exc:
+ raise HTTPException(
+ 502,
+ detail=f"Unable to list resources for connection '{conn.name}': {safe_connection_error(exc)}",
+ ) from exc
+
+
+@router.get("/{connection_id}/resources/{resource:path}/sample")
+def sample_connection_resource(
+ connection_id: str,
+ resource: str,
+ limit: int = Query(20, ge=1, le=1000),
+ db: Session = Depends(get_db),
+ _=Depends(require_editor),
+):
+ """Read a bounded sample from one saved connection resource."""
+ conn = db.query(Connection).filter(Connection.id == connection_id).first()
+ if not conn:
+ raise HTTPException(404, "Connection not found")
+ try:
+ connector = _load_connection_connector(conn, resource=resource)
+ rows = connector.pull_sample(resource, limit=limit)
+ return {"connection_id": connection_id, "resource": resource, "rows": rows}
+ except (ConnectionConfigError, ValueError) as exc:
+ raise HTTPException(422, detail=str(exc)) from exc
+ except Exception as exc:
+ raise HTTPException(
+ 502,
+ detail=f"Unable to read resource '{resource}': {safe_connection_error(exc)}",
+ ) from exc
class RefreshConfigurationUpdate(BaseModel):
diff --git a/backend/app/routers/v2/curated.py b/backend/app/routers/v2/curated.py
index 53ee2cba..2eabe4fe 100644
--- a/backend/app/routers/v2/curated.py
+++ b/backend/app/routers/v2/curated.py
@@ -79,14 +79,34 @@ def delete_curated(dataset_id: str, db: Session = Depends(get_db), _admin=Depend
raise HTTPException(404, "Curated dataset not found")
# 清理关联版本和媒体项
ver_ids = [v.id for v in db.query(DatasetVersion).filter(DatasetVersion.dataset_id == dataset_id).all()]
+ media_uris: list[str] = []
if ver_ids:
+ media_items = db.query(MediaItem).filter(MediaItem.dataset_version_id.in_(ver_ids)).all()
+ for m in media_items:
+ media_uris.append(m.storage_uri)
+ if m.ocr_result_uri:
+ media_uris.append(m.ocr_result_uri)
db.query(MediaItem).filter(MediaItem.dataset_version_id.in_(ver_ids)).delete(synchronize_session=False)
db.query(DatasetVersion).filter(DatasetVersion.dataset_id == dataset_id).delete(synchronize_session=False)
# 清理审核记录
db.query(CuratedReview).filter(CuratedReview.curated_dataset_id == dataset_id).delete(synchronize_session=False)
+ # 若曾发起过审核,_ensure_curated_dataset_row/旧版 /review 端点会用相同 id
+ # 懒创建一条 v2_curated_datasets 镜像行作为 CuratedReview 的 FK 桥接——上面
+ # 只删了 v2_datasets 里的真本体行,这条镜像行没有别的清理入口,不删就永久残留
+ db.query(CuratedDataset).filter(CuratedDataset.id == dataset_id).delete(synchronize_session=False)
db.delete(ds)
db.commit()
+ # Best-effort MinIO cleanup — never fails an already-committed delete
+ if media_uris:
+ from app.services.storage_service import get_storage_service
+ storage = get_storage_service()
+ for uri in media_uris:
+ try:
+ storage.delete_object(uri)
+ except Exception:
+ logger.warning("MinIO cleanup failed for object %s (dataset %s)", uri, dataset_id, exc_info=True)
+
@router.get("/{dataset_id}", response_model=CuratedDatasetResponse)
def get_curated(dataset_id: str, db: Session = Depends(get_db)):
diff --git a/backend/app/routers/v2/datasets.py b/backend/app/routers/v2/datasets.py
index f1811abc..bbc4d273 100644
--- a/backend/app/routers/v2/datasets.py
+++ b/backend/app/routers/v2/datasets.py
@@ -23,6 +23,30 @@ class DatasetResponse(BaseModel):
class Config:
from_attributes = True
+
+_UPLOAD_READ_CHUNK_SIZE = 1024 * 1024
+
+
+async def _read_upload_bounded(
+ file: UploadFile,
+ max_bytes: int,
+ *,
+ chunk_size: int = _UPLOAD_READ_CHUNK_SIZE,
+ error_detail: str | None = None,
+) -> bytes:
+ """Read at most one byte beyond the configured upload limit."""
+ content = bytearray()
+ while len(content) <= max_bytes:
+ read_size = min(chunk_size, max_bytes - len(content) + 1)
+ chunk = await file.read(read_size)
+ if not chunk:
+ break
+ if len(content) + len(chunk) > max_bytes:
+ raise HTTPException(status_code=413, detail=error_detail or "文件超过大小限制")
+ content.extend(chunk)
+ return bytes(content)
+
+
@router.post("/upload", status_code=201)
async def upload_dataset(file: UploadFile = File(...), db: Session = Depends(get_db), _=Depends(require_editor)):
"""上传 CSV/Excel 文件,自动创建 raw Dataset + DatasetVersion"""
@@ -35,9 +59,12 @@ async def upload_dataset(file: UploadFile = File(...), db: Session = Depends(get
if ext not in allowed:
raise HTTPException(400, f"不支持的文件类型: .{ext} (允许: {settings.allowed_upload_extensions})")
- content = await file.read()
- if len(content) > settings.max_upload_mb * 1024 * 1024:
- raise HTTPException(413, f"文件超过大小限制 {settings.max_upload_mb}MB")
+ max_upload_bytes = settings.max_upload_mb * 1024 * 1024
+ content = await _read_upload_bounded(
+ file,
+ max_upload_bytes,
+ error_detail=f"文件超过大小限制 {settings.max_upload_mb}MB",
+ )
# 推断 kind
if ext in ("csv", "xlsx", "xls"):
kind = "structured"
diff --git a/backend/app/routers/v2/graph.py b/backend/app/routers/v2/graph.py
index 7294aafe..c173f9c0 100644
--- a/backend/app/routers/v2/graph.py
+++ b/backend/app/routers/v2/graph.py
@@ -7,6 +7,8 @@
from app.services.publication.working_copy import OntologyWorkingCopyService
from app.services.relation_dedup import dedupe_relation_edges as _dedupe_relation_edges
from app.database import SessionLocal
+from sqlalchemy.orm import Session
+from app.services.ontology_access import GrantDenied, require_project_grant
router = APIRouter(dependencies=[Depends(get_current_user)])
@@ -24,12 +26,26 @@ def get_db():
db.close()
+def _require_read(ontology_id: str, db: Session = Depends(get_db), actor: User = Depends(get_current_user)):
+ try:
+ require_project_grant(db, actor, ontology_id, "read")
+ except GrantDenied as exc:
+ raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc
+
+
+def _require_edit(ontology_id: str, db: Session = Depends(get_db), actor: User = Depends(require_editor)):
+ try:
+ require_project_grant(db, actor, ontology_id, "edit")
+ except GrantDenied as exc:
+ raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc
+
+
class CypherRequest(BaseModel):
query: str
params: dict = {}
-@router.get("/{ontology_id}/graph")
+@router.get("/{ontology_id}/graph", dependencies=[Depends(_require_read)])
def get_graph(ontology_id: str, limit: int = 200, label_filter: str | None = None):
"""返回本体图谱数据 (Neovis.js 兼容格式)"""
svc = get_neo4j()
@@ -95,7 +111,7 @@ def _sqlite_graph_data(ontology_id: str, limit: int = 200, label_filter: str | N
db.close()
-@router.get("/{ontology_id}/graph/quality")
+@router.get("/{ontology_id}/graph/quality", dependencies=[Depends(_require_read)])
def graph_quality(ontology_id: str):
from app.models.entity import Entity
from app.models.relation import Relation
@@ -146,7 +162,7 @@ def graph_quality(ontology_id: str):
db.close()
-@router.get("/{ontology_id}/integrations/status")
+@router.get("/{ontology_id}/integrations/status", dependencies=[Depends(_require_read)])
def integration_status(ontology_id: str):
neo = get_neo4j()
neo_available = neo.available
@@ -161,7 +177,7 @@ def integration_status(ontology_id: str):
}
-@router.post("/{ontology_id}/graph/cypher")
+@router.post("/{ontology_id}/graph/cypher", dependencies=[Depends(_require_read)])
def run_cypher(ontology_id: str, body: CypherRequest):
"""执行 Cypher 查询 (只读校验 + 强制 ontology_id 过滤)"""
from app.services.v2.graph.cypher_builder import validate_readonly_cypher
@@ -180,15 +196,16 @@ def run_cypher(ontology_id: str, body: CypherRequest):
return {"results": results, "neo4j_available": True}
-@router.get("/{ontology_id}/graph/neighbors/{node_id}")
+@router.get("/{ontology_id}/graph/neighbors/{node_id}", dependencies=[Depends(_require_read)])
def get_neighbors(ontology_id: str, node_id: str, depth: int = 1):
"""查询节点邻居"""
svc = get_neo4j()
if not svc.available:
return {"nodes": [], "edges": [], "neo4j_available": False}
query = f"""
- MATCH (n)-[r*1..{min(depth, 5)}]-(m)
+ MATCH path = (n)-[r*1..{min(depth, 5)}]-(m)
WHERE elementId(n) = $node_id AND n.ontology_id = $ontology_id
+ AND all(node IN nodes(path) WHERE node.ontology_id = $ontology_id)
RETURN n, r, m LIMIT 100
"""
results = svc.run_cypher(query, {"node_id": node_id, "ontology_id": ontology_id})
@@ -203,7 +220,7 @@ class NLQueryRequest(BaseModel):
schema: dict = {}
-@router.post("/{ontology_id}/graph/ask")
+@router.post("/{ontology_id}/graph/ask", dependencies=[Depends(_require_read)])
def nl_query(ontology_id: str, body: NLQueryRequest):
"""自然语言 → Cypher → 图数据"""
from app.services.v2.graph.nl2cypher import NL2CypherService
@@ -231,7 +248,7 @@ def nl_query(ontology_id: str, body: NLQueryRequest):
# ── 高级图分析 ─────────────────────────────────────────────────────────
-@router.get("/{ontology_id}/graph/path")
+@router.get("/{ontology_id}/graph/path", dependencies=[Depends(_require_read)])
def graph_path(ontology_id: str, src: str, tgt: str):
"""两节点间最短路径"""
from app.services.v2.graph.graph_analytics import GraphAnalyticsService
@@ -239,7 +256,7 @@ def graph_path(ontology_id: str, src: str, tgt: str):
return svc.shortest_path(ontology_id, src, tgt)
-@router.get("/{ontology_id}/graph/degree/{node_id}")
+@router.get("/{ontology_id}/graph/degree/{node_id}", dependencies=[Depends(_require_read)])
def node_degree(ontology_id: str, node_id: str):
"""查询节点度数(入度 + 出度)"""
from app.services.v2.graph.graph_analytics import GraphAnalyticsService
@@ -247,7 +264,7 @@ def node_degree(ontology_id: str, node_id: str):
return svc.node_degree(ontology_id, node_id)
-@router.get("/{ontology_id}/graph/top-nodes")
+@router.get("/{ontology_id}/graph/top-nodes", dependencies=[Depends(_require_read)])
def top_nodes(ontology_id: str, limit: int = 10):
"""返回连接数最多的 Top-N 节点"""
from app.services.v2.graph.graph_analytics import GraphAnalyticsService
@@ -255,7 +272,7 @@ def top_nodes(ontology_id: str, limit: int = 10):
return {"nodes": svc.top_connected_nodes(ontology_id, limit)}
-@router.post("/{ontology_id}/graph/sync")
+@router.post("/{ontology_id}/graph/sync", dependencies=[Depends(_require_edit)])
def sync_graph(ontology_id: str, current_user: User = Depends(require_editor)):
"""将 SQLite 实体/关系全量同步到 Neo4j"""
from app.database import SessionLocal
diff --git a/backend/app/routers/v2/logic_actions.py b/backend/app/routers/v2/logic_actions.py
index fb22b32c..509d8cf5 100644
--- a/backend/app/routers/v2/logic_actions.py
+++ b/backend/app/routers/v2/logic_actions.py
@@ -12,6 +12,7 @@
from app.models.user import User
from app.models.v2.logic import OntologyLogicRule, OntologyStateMachine
from app.models.v2.action import OntologyActionType, OntologyActionRun
+from app.services.ontology_access import GrantDenied, require_project_grant
router = APIRouter(dependencies=[Depends(get_current_user)])
@@ -23,6 +24,20 @@ def get_db():
db.close()
+def _require_read(ontology_id: str, db: Session = Depends(get_db), actor: User = Depends(get_current_user)):
+ try:
+ require_project_grant(db, actor, ontology_id, "read")
+ except GrantDenied as exc:
+ raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc
+
+
+def _require_edit(ontology_id: str, db: Session = Depends(get_db), actor: User = Depends(require_editor)):
+ try:
+ require_project_grant(db, actor, ontology_id, "edit")
+ except GrantDenied as exc:
+ raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc
+
+
# ── Logic Rules ─────────────────────────────────────────────────
class LogicRuleCreate(BaseModel):
@@ -59,7 +74,7 @@ class LogicTestRequest(BaseModel):
parameters: dict = {}
-@router.get("/{ontology_id}/logic")
+@router.get("/{ontology_id}/logic", dependencies=[Depends(_require_read)])
def list_logic_rules(ontology_id: str, logic_type: str = "", db: Session = Depends(get_db)):
q = db.query(OntologyLogicRule).filter(OntologyLogicRule.ontology_id == ontology_id)
if logic_type:
@@ -73,7 +88,7 @@ def list_logic_rules(ontology_id: str, logic_type: str = "", db: Session = Depen
for r in rules]
-@router.post("/{ontology_id}/logic", status_code=201)
+@router.post("/{ontology_id}/logic", status_code=201, dependencies=[Depends(_require_edit)])
def create_logic_rule(ontology_id: str, body: LogicRuleCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
def _write():
rule = OntologyLogicRule(
@@ -87,7 +102,7 @@ def _write():
return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2logic.create", callback=_write)
-@router.put("/{ontology_id}/logic/{rule_id}")
+@router.put("/{ontology_id}/logic/{rule_id}", dependencies=[Depends(_require_edit)])
def update_logic_rule(ontology_id: str, rule_id: str, body: LogicRuleCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
rule = db.query(OntologyLogicRule).filter(
OntologyLogicRule.id == rule_id, OntologyLogicRule.ontology_id == ontology_id
@@ -102,7 +117,7 @@ def _write():
return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2logic.update", callback=_write)
-@router.patch("/{ontology_id}/logic/{rule_id}")
+@router.patch("/{ontology_id}/logic/{rule_id}", dependencies=[Depends(_require_edit)])
def patch_logic_rule(ontology_id: str, rule_id: str, body: LogicRulePatch, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
rule = db.query(OntologyLogicRule).filter(
OntologyLogicRule.id == rule_id, OntologyLogicRule.ontology_id == ontology_id
@@ -121,7 +136,7 @@ def _write():
return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2logic.patch", callback=_write)
-@router.post("/{ontology_id}/logic/{rule_id}/review")
+@router.post("/{ontology_id}/logic/{rule_id}/review", dependencies=[Depends(_require_edit)])
def review_logic_rule(ontology_id: str, rule_id: str, body: LogicReviewRequest, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
rule = db.query(OntologyLogicRule).filter(
OntologyLogicRule.id == rule_id, OntologyLogicRule.ontology_id == ontology_id
@@ -147,7 +162,7 @@ def _write():
return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2logic.review", callback=_write)
-@router.post("/{ontology_id}/logic/{rule_id}/test")
+@router.post("/{ontology_id}/logic/{rule_id}/test", dependencies=[Depends(_require_read)])
def test_logic_rule(ontology_id: str, rule_id: str, body: LogicTestRequest, db: Session = Depends(get_db), _=Depends(require_editor)):
rule = db.query(OntologyLogicRule).filter(
OntologyLogicRule.id == rule_id, OntologyLogicRule.ontology_id == ontology_id
@@ -157,7 +172,7 @@ def test_logic_rule(ontology_id: str, rule_id: str, body: LogicTestRequest, db:
return {"rule_id": rule.id, **_evaluate_logic_rule(rule, body.row or {}, body.parameters or {})}
-@router.delete("/{ontology_id}/logic/{rule_id}")
+@router.delete("/{ontology_id}/logic/{rule_id}", dependencies=[Depends(_require_edit)])
def delete_logic_rule(ontology_id: str, rule_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
rule = db.query(OntologyLogicRule).filter(
OntologyLogicRule.id == rule_id, OntologyLogicRule.ontology_id == ontology_id
@@ -170,7 +185,7 @@ def _write():
return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2logic.delete", callback=_write)
-@router.post("/{ontology_id}/logic/publish")
+@router.post("/{ontology_id}/logic/publish", dependencies=[Depends(_require_edit)])
def publish_logic_rules_v2(ontology_id: str, db: Session = Depends(get_db), _=Depends(require_editor)):
"""Direct tool publication is closed; use the Ontology compiler (P1C)."""
raise HTTPException(status_code=403, detail="PUBLICATION_NOT_ENABLED")
@@ -197,7 +212,7 @@ def publish_logic_rules_v2(ontology_id: str, db: Session = Depends(get_db), _=De
# ── Logic: Discovery ────────────────────────────────────────────
-@router.post("/{ontology_id}/logic/discover")
+@router.post("/{ontology_id}/logic/discover", dependencies=[Depends(_require_edit)])
def discover_logic_rules(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
"""发现 Logic Rules(同步写入 v2 + v1 表,供前端 LogicTab 读取)"""
from app.services.v2.mapping.mapping_service import MappingService
@@ -268,7 +283,7 @@ def _result():
# ── State Machines ──────────────────────────────────────────────
-@router.get("/{ontology_id}/state-machines")
+@router.get("/{ontology_id}/state-machines", dependencies=[Depends(_require_read)])
def list_state_machines(ontology_id: str, db: Session = Depends(get_db)):
machines = db.query(OntologyStateMachine).filter(
OntologyStateMachine.ontology_id == ontology_id
@@ -319,7 +334,7 @@ class ActionReviewRequest(BaseModel):
notes: str = ""
-@router.get("/{ontology_id}/actions")
+@router.get("/{ontology_id}/actions", dependencies=[Depends(_require_read)])
def list_action_types(ontology_id: str, category: str = "", db: Session = Depends(get_db)):
q = db.query(OntologyActionType).filter(OntologyActionType.ontology_id == ontology_id)
if category:
@@ -334,7 +349,7 @@ def list_action_types(ontology_id: str, category: str = "", db: Session = Depend
"created_at": a.created_at.isoformat() if a.created_at else None} for a in actions]
-@router.post("/{ontology_id}/actions", status_code=201)
+@router.post("/{ontology_id}/actions", status_code=201, dependencies=[Depends(_require_edit)])
def create_action_type(ontology_id: str, body: ActionTypeCreate, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
def _write():
act = OntologyActionType(
@@ -349,7 +364,7 @@ def _write():
return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2action.create", callback=_write)
-@router.patch("/{ontology_id}/actions/{action_id}")
+@router.patch("/{ontology_id}/actions/{action_id}", dependencies=[Depends(_require_edit)])
def patch_action_type(ontology_id: str, action_id: str, body: ActionTypePatch, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
act = db.query(OntologyActionType).filter(
OntologyActionType.id == action_id, OntologyActionType.ontology_id == ontology_id
@@ -368,7 +383,7 @@ def _write():
return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2action.patch", callback=_write)
-@router.post("/{ontology_id}/actions/{action_id}/review")
+@router.post("/{ontology_id}/actions/{action_id}/review", dependencies=[Depends(_require_edit)])
def review_action_type(ontology_id: str, action_id: str, body: ActionReviewRequest, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
act = db.query(OntologyActionType).filter(
OntologyActionType.id == action_id, OntologyActionType.ontology_id == ontology_id
@@ -399,7 +414,7 @@ def _write():
return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2action.review", callback=_write)
-@router.post("/{ontology_id}/actions/discover")
+@router.post("/{ontology_id}/actions/discover", dependencies=[Depends(_require_edit)])
def discover_actions(ontology_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
"""发现 Actions(同步写入 v2 + v1 表,供前端 ActionsTab 读取)"""
from app.services.v2.mapping.mapping_service import MappingService
@@ -457,7 +472,7 @@ def _result():
return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2action.discover", callback=_result)
-@router.delete("/{ontology_id}/actions/{action_id}")
+@router.delete("/{ontology_id}/actions/{action_id}", dependencies=[Depends(_require_edit)])
def delete_action_type(ontology_id: str, action_id: str, db: Session = Depends(get_db), current_user: User = Depends(require_editor)):
act = db.query(OntologyActionType).filter(
OntologyActionType.id == action_id, OntologyActionType.ontology_id == ontology_id
@@ -470,7 +485,7 @@ def _write():
return OntologyWorkingCopyService.mutate(db, ontology_id=ontology_id, actor_id=current_user.id, operation="v2action.delete", callback=_write)
-@router.post("/{ontology_id}/actions/publish")
+@router.post("/{ontology_id}/actions/publish", dependencies=[Depends(_require_edit)])
def publish_actions_v2(ontology_id: str, db: Session = Depends(get_db), _=Depends(require_editor)):
"""Direct tool publication is closed; use the Ontology compiler (P1C)."""
raise HTTPException(status_code=403, detail="PUBLICATION_NOT_ENABLED")
@@ -497,7 +512,7 @@ def publish_actions_v2(ontology_id: str, db: Session = Depends(get_db), _=Depend
# ── Action Runs ─────────────────────────────────────────────────
-@router.get("/{ontology_id}/action-runs")
+@router.get("/{ontology_id}/action-runs", dependencies=[Depends(_require_read)])
def list_action_runs(ontology_id: str, limit: int = 20, db: Session = Depends(get_db)):
runs = db.query(OntologyActionRun).filter(
OntologyActionRun.ontology_id == ontology_id
@@ -507,7 +522,7 @@ def list_action_runs(ontology_id: str, limit: int = 20, db: Session = Depends(ge
"started_at": r.started_at.isoformat() if r.started_at else None} for r in runs]
-@router.post("/{ontology_id}/actions/{action_id}/run")
+@router.post("/{ontology_id}/actions/{action_id}/run", dependencies=[Depends(_require_edit)])
def run_action_type(
ontology_id: str,
action_id: str,
diff --git a/backend/app/routers/v2/pipelines.py b/backend/app/routers/v2/pipelines.py
index 5441dcd2..12f37211 100644
--- a/backend/app/routers/v2/pipelines.py
+++ b/backend/app/routers/v2/pipelines.py
@@ -4,6 +4,7 @@
from pydantic import BaseModel
from sqlalchemy.orm import Session
from typing import Optional
+import copy
from datetime import datetime, timezone
from app.database import SessionLocal
from app.deps import get_current_user, require_admin, require_editor
@@ -22,6 +23,80 @@ def get_db():
db.close()
+_SQL_SOURCE_TYPES = {"mysql", "postgres", "postgresql"}
+_UNSUPPORTED_INLINE_SOURCE_TYPES = {"mongo", "mongodb", "rest", "rest_api"}
+_UNSUPPORTED_INLINE_CONNECTOR_CODE = "UNSUPPORTED_INLINE_CONNECTOR"
+
+
+def _unsupported_inline_connector_nodes(definition: Optional[dict]) -> list[tuple[str, str]]:
+ if not isinstance(definition, dict):
+ return []
+ nodes = definition.get("nodes", [])
+ if not isinstance(nodes, list):
+ return []
+ unsupported = []
+ for node in nodes:
+ if not isinstance(node, dict) or node.get("type") != "connector":
+ continue
+ config = node.get("config")
+ if not isinstance(config, dict):
+ continue
+ source_type = str(config.get("source_type", "")).strip().lower()
+ if source_type in _UNSUPPORTED_INLINE_SOURCE_TYPES:
+ unsupported.append((str(node.get("id", "")), source_type))
+ return unsupported
+
+
+def _reject_unsupported_inline_connectors(definition: Optional[dict]) -> None:
+ if _unsupported_inline_connector_nodes(definition):
+ raise HTTPException(status_code=422, detail=_UNSUPPORTED_INLINE_CONNECTOR_CODE)
+
+
+def _sanitize_definition(definition: Optional[dict]) -> Optional[dict]:
+ """Keep only saved-connection references for SQL connector nodes.
+
+ Credentials come from a saved Connection at run time. Legacy inline
+ values are not needed by the worker and must never be persisted again.
+ """
+ if definition is None:
+ return None
+ sanitized = copy.deepcopy(definition)
+ nodes = sanitized.get("nodes", []) if isinstance(sanitized, dict) else []
+ if not isinstance(nodes, list):
+ return sanitized
+ for node in nodes:
+ if not isinstance(node, dict) or node.get("type") != "connector":
+ continue
+ config = node.get("config")
+ if not isinstance(config, dict):
+ continue
+ source_type = str(config.get("source_type", "")).lower()
+ if source_type not in _SQL_SOURCE_TYPES:
+ continue
+ node["config"] = {
+ key: value for key, value in config.items()
+ if key in {"source_type", "connection_id", "resource", "table", "sync_mode"}
+ }
+ return sanitized
+
+
+def _public_definition(definition: Optional[dict]) -> Optional[dict]:
+ """Hide credentials in legacy definitions without mutating stored rows."""
+ sanitized = _sanitize_definition(definition)
+ if not isinstance(sanitized, dict):
+ return sanitized
+ for node in sanitized.get("nodes", []):
+ if not isinstance(node, dict) or node.get("type") != "connector":
+ continue
+ config = node.get("config")
+ if not isinstance(config, dict):
+ continue
+ source_type = str(config.get("source_type", "")).strip().lower()
+ if source_type in _UNSUPPORTED_INLINE_SOURCE_TYPES:
+ node["config"] = {"source_type": source_type}
+ return sanitized
+
+
# ── Pydantic Models ────────────────────────────────────────────────
class PipelineCreate(BaseModel):
@@ -76,6 +151,7 @@ class ValidateResult(BaseModel):
@router.post("", response_model=PipelineResponse, status_code=201, dependencies=[Depends(require_editor)])
def create_pipeline(body: PipelineCreate, db: Session = Depends(get_db)):
"""创建新 Pipeline。支持旧 steps 格式和新 nodes/edges DSL。"""
+ _reject_unsupported_inline_connectors(body.definition)
# 重名校验
existing = db.query(Pipeline).filter(
Pipeline.name == body.name,
@@ -100,7 +176,7 @@ def create_pipeline(body: PipelineCreate, db: Session = Depends(get_db)):
source_dataset_id=body.source_dataset_id,
route=inferred_route or "A", # SQLite 该列有 NOT NULL 约束
spec=body.spec or {},
- definition=body.definition,
+ definition=_sanitize_definition(body.definition),
status="draft",
branch="main",
version=1,
@@ -159,8 +235,11 @@ def update_pipeline(pipeline_id: str, body: PipelineUpdate, db: Session = Depend
if not pl:
raise HTTPException(404, "Pipeline not found")
+ _reject_unsupported_inline_connectors(body.definition)
update_data = body.model_dump(exclude_unset=True)
for k, v in update_data.items():
+ if k == "definition":
+ v = _sanitize_definition(v)
setattr(pl, k, v)
pl.updated_at = datetime.now(timezone.utc)
db.commit()
@@ -201,6 +280,14 @@ def validate_pipeline(pipeline_id: str, db: Session = Depends(get_db), _=Depends
nodes = definition.get("nodes", [])
edges = definition.get("edges", [])
+ for node_id, source_type in _unsupported_inline_connector_nodes(definition):
+ errors.append({
+ "node_id": node_id,
+ "severity": "error",
+ "code": _UNSUPPORTED_INLINE_CONNECTOR_CODE,
+ "message": "MongoDB/REST inline connector nodes are unsupported in Builder; use File Upload in Builder and manage external connections in Data Connections.",
+ })
+
if not nodes:
errors.append({"node_id": "", "severity": "error", "message": "Pipeline 至少需要一个节点。"})
@@ -320,6 +407,7 @@ def run_pipeline(pipeline_id: str, db: Session = Depends(get_db), _=Depends(requ
pl = db.query(Pipeline).filter(Pipeline.id == pipeline_id).first()
if not pl:
raise HTTPException(404, "Pipeline not found")
+ _reject_unsupported_inline_connectors(pl.definition)
pl.status = "running"
run = PipelineRun(pipeline_id=pipeline_id, status="pending", started_at=datetime.now(timezone.utc))
@@ -453,7 +541,7 @@ def _format_pipeline(pl: Pipeline) -> dict:
"source_dataset_id": pl.source_dataset_id,
"route": pl.route,
"spec": pl.spec or {},
- "definition": pl.definition,
+ "definition": _public_definition(pl.definition),
"status": pl.status or "draft",
"branch": pl.branch or "main",
"version": pl.version or 1,
diff --git a/backend/app/routers/v2/search.py b/backend/app/routers/v2/search.py
index 673fc31d..6fcaf540 100644
--- a/backend/app/routers/v2/search.py
+++ b/backend/app/routers/v2/search.py
@@ -1,13 +1,15 @@
"""v2 Search API — 关键词/语义统一搜索"""
from __future__ import annotations
import json
-from fastapi import APIRouter, Depends, Query
+from fastapi import APIRouter, Depends, HTTPException, Query
from pydantic import BaseModel
from sqlalchemy import cast, or_, String as SAString
from sqlalchemy.orm import Session
from app.database import SessionLocal
from app.deps import get_current_user
from app.models.entity import Entity
+from app.models.user import User
+from app.services.ontology_access import GrantDenied, require_project_grant
router = APIRouter(dependencies=[Depends(get_current_user)])
@@ -20,6 +22,13 @@ def get_db():
db.close()
+def _require_read(ontology_id: str, db: Session = Depends(get_db), actor: User = Depends(get_current_user)):
+ try:
+ require_project_grant(db, actor, ontology_id, "read")
+ except GrantDenied as exc:
+ raise HTTPException(status_code=404, detail="ONTOLOGY_NOT_FOUND") from exc
+
+
class SearchRequest(BaseModel):
query: str
mode: str = "keyword" # keyword | semantic
@@ -54,7 +63,7 @@ def _sql_keyword_search(db: Session, ontology_id: str, q: str, n: int) -> list[d
]
-@router.get("/{ontology_id}/search/keyword")
+@router.get("/{ontology_id}/search/keyword", dependencies=[Depends(_require_read)])
def keyword_search(
ontology_id: str,
q: str = Query(..., description="搜索词"),
@@ -71,7 +80,7 @@ def keyword_search(
return {"results": results, "chroma_available": True, "query": q}
-@router.get("/{ontology_id}/search/semantic")
+@router.get("/{ontology_id}/search/semantic", dependencies=[Depends(_require_read)])
def semantic_search(
ontology_id: str,
q: str = Query(..., description="搜索词"),
@@ -87,7 +96,7 @@ def semantic_search(
return {"results": results, "chroma_available": True, "query": q}
-@router.post("/{ontology_id}/search")
+@router.post("/{ontology_id}/search", dependencies=[Depends(_require_read)])
def unified_search(ontology_id: str, body: SearchRequest, db: Session = Depends(get_db)):
"""统一搜索端点"""
from app.services.v2.vector.chroma_service import ChromaService
diff --git a/backend/app/schemas/ontology.py b/backend/app/schemas/ontology.py
index 1c9906e8..e284d559 100644
--- a/backend/app/schemas/ontology.py
+++ b/backend/app/schemas/ontology.py
@@ -33,6 +33,7 @@ class OntologyOut(BaseModel):
version: str
status: str
build_mode: Optional[str] = "simple_llm"
+ is_dirty: bool = False
created_by: str
created_at: datetime
updated_at: datetime
diff --git a/backend/app/services/connection/config.py b/backend/app/services/connection/config.py
new file mode 100644
index 00000000..c865d421
--- /dev/null
+++ b/backend/app/services/connection/config.py
@@ -0,0 +1,96 @@
+"""Shared connection configuration handling.
+
+Connection records keep their credentials inside an encrypted JSON envelope.
+Every runtime path that opens a source database goes through this module so a
+saved connection and a refresh worker build the same SQLAlchemy URL.
+"""
+from __future__ import annotations
+
+import json
+from typing import Any, Mapping
+from urllib.parse import quote
+
+
+SQL_KINDS = {"mysql", "postgres", "postgresql"}
+
+
+class ConnectionConfigError(ValueError):
+ """A saved connection could not be converted into runtime configuration."""
+
+
+def decrypt_connection_config(raw_config: Mapping[str, Any] | None) -> dict[str, Any]:
+ """Return a runtime config from either plaintext or an encrypted envelope.
+
+ Plaintext is accepted for compatibility with old fixtures and direct
+ connector callers. Persisted API writes always use the encrypted form.
+ """
+ raw = dict(raw_config or {})
+ encrypted = raw.get("_encrypted")
+ if not encrypted:
+ return raw
+ try:
+ from app.services import encryption_service
+
+ decoded = json.loads(encryption_service.decrypt(str(encrypted)))
+ except Exception as exc:
+ raise ConnectionConfigError("connection configuration could not be decrypted") from exc
+ if not isinstance(decoded, dict):
+ raise ConnectionConfigError("connection configuration must be a JSON object")
+ return decoded
+
+
+def _sql_url(raw_config: Mapping[str, Any], kind: str) -> str:
+ existing = raw_config.get("connection_string")
+ if existing:
+ return str(existing)
+
+ host = str(raw_config.get("host") or "localhost")
+ port = raw_config.get("port") or (3306 if kind == "mysql" else 5432)
+ try:
+ port_value = int(port)
+ except (TypeError, ValueError) as exc:
+ raise ConnectionConfigError("database port must be a number") from exc
+ if not 1 <= port_value <= 65535:
+ raise ConnectionConfigError("database port must be between 1 and 65535")
+
+ user = str(raw_config.get("user") or "")
+ password = str(raw_config.get("password") or "")
+ database = str(raw_config.get("database") or "")
+ if not user or not database:
+ raise ConnectionConfigError("database user and database name are required")
+
+ scheme = "mysql+pymysql" if kind == "mysql" else "postgresql+psycopg2"
+ # Keep the host as supplied (including bracketed IPv6 literals), while
+ # encoding URL components that may contain @, :, /, #, or whitespace.
+ return (
+ f"{scheme}://{quote(user, safe='')}:{quote(password, safe='')}"
+ f"@{host}:{port_value}/{quote(database, safe='')}"
+ )
+
+
+def build_connector_config(
+ kind: str,
+ raw_config: Mapping[str, Any] | None,
+ *,
+ resource: str | None = None,
+) -> dict[str, Any]:
+ """Resolve persisted config and add the runtime DSN for SQL sources."""
+ normalized_kind = "postgres" if kind == "postgresql" else kind
+ config = decrypt_connection_config(raw_config)
+ if normalized_kind in SQL_KINDS:
+ config["connection_string"] = _sql_url(config, normalized_kind)
+ if resource is not None:
+ config["resource"] = resource
+ return config
+
+
+def safe_connection_error(exc: BaseException) -> str:
+ """Return useful connector detail while redacting common URL secrets."""
+ message = str(exc) or exc.__class__.__name__
+ # SQLAlchemy normally masks passwords, but drivers occasionally include a
+ # raw DSN in their exception. Avoid leaking password material in API
+ # responses and logs when that happens.
+ import re
+
+ message = re.sub(r"(://[^:/@]+:)[^@]+(@)", r"\1***\2", message)
+ return message[:500]
diff --git a/backend/app/services/connection/sql_connector.py b/backend/app/services/connection/sql_connector.py
index c15b0abe..7b54171e 100644
--- a/backend/app/services/connection/sql_connector.py
+++ b/backend/app/services/connection/sql_connector.py
@@ -8,6 +8,7 @@
from sqlalchemy import create_engine, inspect, text
from app.services.connection.base import ConnectorBase, DeltaPage, records_to_delta_page
+from app.services.connection.config import safe_connection_error
from app.schemas.refresh import SourceCursor
# 合法 SQL 标识符:字母、数字、下划线、点(schema.table);禁止空格/分号/引号/注释等
@@ -36,6 +37,7 @@ class SQLConnector(ConnectorBase):
def __init__(self, config: dict):
self._config = config
self._engine = None
+ self.last_error: str | None = None
def _get_engine(self):
if self._engine is None:
@@ -50,8 +52,10 @@ def test_connection(self) -> bool:
try:
with self._get_engine().connect() as conn:
conn.execute(text("SELECT 1"))
+ self.last_error = None
return True
- except Exception:
+ except Exception as exc:
+ self.last_error = safe_connection_error(exc)
return False
def list_resources(self) -> list[str]:
diff --git a/backend/app/services/graph_diagnostics.py b/backend/app/services/graph_diagnostics.py
new file mode 100644
index 00000000..639ca470
--- /dev/null
+++ b/backend/app/services/graph_diagnostics.py
@@ -0,0 +1,68 @@
+"""Graph connectivity diagnostics (graph-engineering playbook, Section V.E):
+after entity resolution, a few structural signals reveal whether resolution
+actually worked. A single connected component means every entity is
+reachable from every other — fragmented islands mean surface-form variants
+that should have merged did not. The edge/node ratio gives a density read;
+the highest-degree entities are the hub nodes a future summarization stage
+would prioritize.
+
+Pure functions only — no database access, no LLM calls — so this is callable
+both from the extraction pipeline and, later, on demand for any ontology.
+"""
+
+
+def compute_graph_diagnostics(entities: list[dict], edges: list[tuple[str, str]], top_n: int = 5) -> dict:
+ """`entities`: [{"id": ..., "name_cn": ...}, ...] every entity in the
+ ontology. `edges`: [(source_id, target_id), ...] every relation,
+ direction ignored (weak connectivity, per the playbook)."""
+ ids = [e["id"] for e in entities if e.get("id")]
+ names = {e["id"]: e.get("name_cn") or e["id"] for e in entities if e.get("id")}
+ id_set = set(ids)
+
+ adjacency: dict[str, list[str]] = {i: [] for i in ids}
+ degree: dict[str, int] = {i: 0 for i in ids}
+ edge_count = 0
+ for source, target in edges:
+ if source not in id_set or target not in id_set:
+ continue # a dangling reference is a separate concern (P0 validation)
+ adjacency[source].append(target)
+ adjacency[target].append(source)
+ degree[source] += 1
+ degree[target] += 1
+ edge_count += 1
+
+ components = _weakly_connected_components(ids, adjacency)
+ component_sizes = sorted((len(c) for c in components), reverse=True)
+ isolated = [i for i in ids if degree[i] == 0]
+ hubs = sorted(ids, key=lambda i: degree[i], reverse=True)[:top_n]
+
+ return {
+ "entity_count": len(ids),
+ "edge_count": edge_count,
+ "edge_node_ratio": round(edge_count / len(ids), 2) if ids else 0.0,
+ "connected_components": len(components),
+ "largest_component_size": component_sizes[0] if component_sizes else 0,
+ "isolated_entity_count": len(isolated),
+ "isolated_entities": [names[i] for i in isolated[:top_n]],
+ "hub_entities": [{"name_cn": names[i], "degree": degree[i]} for i in hubs if degree[i] > 0],
+ }
+
+
+def _weakly_connected_components(ids: list[str], adjacency: dict[str, list[str]]) -> list[set]:
+ seen: set[str] = set()
+ components: list[set] = []
+ for start in ids:
+ if start in seen:
+ continue
+ component = {start}
+ stack = [start]
+ seen.add(start)
+ while stack:
+ node = stack.pop()
+ for neighbor in adjacency.get(node, []):
+ if neighbor not in seen:
+ seen.add(neighbor)
+ component.add(neighbor)
+ stack.append(neighbor)
+ components.append(component)
+ return components
diff --git a/backend/app/services/llm_service.py b/backend/app/services/llm_service.py
index 7f9bfdf6..5575ad32 100644
--- a/backend/app/services/llm_service.py
+++ b/backend/app/services/llm_service.py
@@ -31,7 +31,12 @@ def extract_ontology(text: str, prompt_content: str, model_config: dict, model_n
"每条关系的 source/target 必须取自已提取的实体\n"
"5. 逻辑规则按 Palantir Ontology Functions 模型分类为 derived_property/aggregation/"
"complex_edit/external_query 之一,写入 function_type + definition 字段,禁止输出 "
- "IF-THEN 自然语言公式\n\n"
+ "IF-THEN 自然语言公式\n"
+ "6. 业务规则、政策、判断条件、定价/审批/校验类规则(如\"促销定价规则\")不是实体,"
+ "禁止作为 entity 或 instance 输出,必须放入 logic_rules 数组\n"
+ "7. 属性的枚举值/分级/分档/阶段标签(如信用分层的\"A层\"\"B层\"、逾期阶段的\"M0\"\"M1\"、"
+ "融资轮次的\"A轮融资\")不是独立概念也不是命名实例,不要为它们创建 entity 或 instance,"
+ "只需体现为所属概念某个属性的枚举值\n\n"
f"文档内容:\n\n{text}"
)},
]
@@ -61,7 +66,7 @@ def normalize_extracted_ontology(parsed: Any) -> dict:
if not isinstance(entity, dict):
continue
name = _clean_entity_name(entity.get("name_cn"))
- if not name or _is_junk_entity_name(name):
+ if not name or _is_junk_entity_name(name) or _looks_like_rule_name(name) or _looks_like_enum_value(name):
continue
key = _normalize_name(name)
canonical = seen_names.get(key)
@@ -140,14 +145,47 @@ def _is_junk_entity_name(name: str) -> bool:
return False
-def resolve_entities(entities: list[dict], model_config: dict, model_name: str) -> dict[str, str]:
+_RULE_NAME_SUFFIXES = ("规则", "政策", "办法", "制度", "准则", "细则", "规程")
+
+
+def _looks_like_rule_name(name: str) -> bool:
+ """A business rule/policy (e.g. "促销定价规则") is not an entity — it
+ belongs in logic_rules, not the concept graph."""
+ return name.endswith(_RULE_NAME_SUFFIXES)
+
+
+# A letter (+ optional 1-2 digits) followed by a small closed set of
+# tier/stage/round markers, and nothing else — the shape of an enumeration
+# VALUE of some concept's property (信用分层's "A层", 逾期阶段's "M0"/"M1+",
+# 融资轮次's "D轮融资"), not a concept in its own right.
+_ENUM_VALUE_PATTERN = re.compile(
+ r"^[A-Za-z]{1,2}[+-]?(轮融资|阶段|层|级|档|类)$|^M\d{1,2}\+?(阶段)?$"
+)
+
+
+def _looks_like_enum_value(name: str) -> bool:
+ return bool(_ENUM_VALUE_PATTERN.match(name))
+
+
+def resolve_entities(
+ entities: list[dict], model_config: dict, model_name: str,
+ existing_entities: list[dict] | None = None,
+) -> dict[str, str]:
"""Cross-document entity resolution (graph-engineering playbook, resolution
stage): within each entity type, cluster surface-form variants of the same
real-world concept using the extraction-time descriptions as disambiguation
context — catches cases exact-name matching misses (abbreviation vs full
name, alias vs formal name across files). Returns an alias map
{raw_name_cn: canonical_name_cn} covering every input name; unmatched names
- and failed calls fall back to identity so no entity is ever silently lost."""
+ and failed calls fall back to identity so no entity is ever silently lost.
+
+ `existing_entities` (playbook incremental-update guidance: resolve new
+ entities against the existing canonical set, not against each other) are
+ entities already saved on this ontology from a prior run. They are never
+ renamed — if a new entity clusters with one of them, that existing
+ entity's name_cn is forced as the canonical, so re-running extraction
+ never silently relabels an already-published entity out from under its
+ existing relations/instances."""
alias_map: dict[str, str] = {}
by_type: dict[str, list[dict]] = {}
for e in entities:
@@ -157,21 +195,43 @@ def resolve_entities(entities: list[dict], model_config: dict, model_name: str)
alias_map[name] = name
by_type.setdefault(e.get("type") or "", []).append(e)
+ existing_by_type: dict[str, list[dict]] = {}
+ existing_names: set[str] = set()
+ for e in existing_entities or []:
+ name = e.get("name_cn")
+ if not name:
+ continue
+ existing_names.add(name)
+ existing_by_type.setdefault(e.get("type") or "", []).append(e)
+
provider = model_config.get("provider", "openai")
api_key = model_config.get("api_key", "")
api_base = model_config.get("api_base")
for etype, group in by_type.items():
- if len(group) < 2:
+ existing_group = existing_by_type.get(etype, [])
+ if len(group) + len(existing_group) < 2:
continue
entity_list = "\n".join(
f"- {e['name_cn']}: {(e.get('description') or '').strip()[:100]}" for e in group
)
+ if existing_group:
+ entity_list += "\n" + "\n".join(
+ f"- [已存在] {e['name_cn']}: {(e.get('description') or '').strip()[:100]}"
+ for e in existing_group
+ )
+ incremental_rule = (
+ "\n标记为 [已存在] 的实体是本体里已经发布的规范实体:如果某个未标记的新实体与其属于"
+ "同一概念,该簇的 canonical 必须使用 [已存在] 实体的名字(去掉标记),绝不能改名或"
+ "另选一个新名字。"
+ if existing_group else ""
+ )
system_prompt = (
f"你是实体消歧专家。下面是同一类型({etype or '未分类'})从文档中提取的实体,"
"其中部分可能是同一概念的不同写法(如全称/简称、中英文并列、别名)。请聚类:"
"每个输入名称必须出现在且仅出现在一个簇的 aliases 中;确实不同的概念各自单独成簇;"
- "结合描述判断,不要仅凭字面相似合并不同概念;canonical 取信息最完整、无歧义的写法。\n\n"
+ "结合描述判断,不要仅凭字面相似合并不同概念;canonical 取信息最完整、无歧义的写法。"
+ f"{incremental_rule}\n\n"
'只返回 JSON:{"clusters": [{"canonical": "规范名", "aliases": ["写法1", "写法2"]}]}'
)
messages = [
@@ -183,15 +243,25 @@ def resolve_entities(entities: list[dict], model_config: dict, model_name: str)
except Exception:
continue # resolution is best-effort; keep identity mapping on failure
group_names = {e["name_cn"] for e in group}
+ existing_group_names = {e["name_cn"] for e in existing_group}
+ all_names = group_names | existing_group_names
for cluster in (parsed.get("clusters") or []) if isinstance(parsed, dict) else []:
if not isinstance(cluster, dict):
continue
canonical = _clean_entity_name(cluster.get("canonical"))
- if not canonical or canonical not in group_names:
+ aliases = [_clean_entity_name(a) for a in (cluster.get("aliases") or [])]
+ aliases = [a for a in aliases if a and a in all_names]
+ # an existing (already-published) entity in this cluster always
+ # wins the canonical slot, regardless of what the model picked
+ existing_in_cluster = [a for a in aliases if a in existing_group_names] + (
+ [canonical] if canonical in existing_group_names else []
+ )
+ if existing_in_cluster:
+ canonical = existing_in_cluster[0]
+ elif not canonical or canonical not in group_names:
continue
- for alias in cluster.get("aliases") or []:
- alias = _clean_entity_name(alias)
- if alias and alias in group_names:
+ for alias in aliases:
+ if alias in group_names: # only remap names from this run's own entities
alias_map[alias] = canonical
return alias_map
@@ -319,6 +389,51 @@ def infer_relations(entities: list, existing_relations: list, text: str,
return [] # relation inference failure is non-fatal
+def summarize_entity(
+ name: str, description: str, relations_text: str, model_config: dict, model_name: str,
+) -> dict | None:
+ """Hub-node profile synthesis (graph-engineering playbook, Section V.A):
+ for a high-degree entity, synthesize a 2-3 paragraph summary, 3-5 atomic
+ traceable key facts, and a time range from its grounded description and
+ its known graph relations. Returns None on failure — summarization is
+ best-effort and never blocks extraction."""
+ provider = model_config.get("provider", "openai")
+ api_key = model_config.get("api_key", "")
+ api_base = model_config.get("api_base")
+
+ system_prompt = (
+ f"为本体实体「{name}」生成知识图谱画像。\n\n"
+ f"已知描述:{description or '(无)'}\n\n"
+ f"该实体在图中的已知关系:\n{relations_text or '(无)'}\n\n"
+ "基于以上信息写一段2-3段的事实性综合描述;如信息有冲突,优先采用更具体的说法;"
+ "提炼3-5条可溯源到上述描述/关系的原子关键事实;不要编造描述和关系之外没有支持的事实;"
+ "时间范围用 YYYY 或 YYYY-MM 格式,没有明确时间信息就填 \"unknown\"。\n\n"
+ '只返回 JSON:{"summary": "...", "key_facts": ["...", "..."], '
+ '"time_range": {"start": "...", "end": "..."}}'
+ )
+ messages = [
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": f"实体:{name}"},
+ ]
+ try:
+ parsed = _parse_response(_call_llm(provider, api_key, api_base, model_name, messages))
+ except Exception:
+ return None
+ if not isinstance(parsed, dict):
+ return None
+ time_range = parsed.get("time_range")
+ if not isinstance(time_range, dict):
+ time_range = {"start": "unknown", "end": "unknown"}
+ return {
+ "summary": str(parsed.get("summary") or ""),
+ "key_facts": [f for f in (parsed.get("key_facts") or []) if isinstance(f, str)][:5],
+ "time_range": {
+ "start": str(time_range.get("start") or "unknown"),
+ "end": str(time_range.get("end") or "unknown"),
+ },
+ }
+
+
def _call_llm(provider: str, api_key: str, api_base: str | None, model: str, messages: list, json_mode: bool = True) -> str:
# Stable seed for reproducibility: derived from message content so same input → same seed.
import hashlib as _hashlib, json as _json
diff --git a/backend/app/services/ontology_access.py b/backend/app/services/ontology_access.py
index 6fa7feea..a25b5fdb 100644
--- a/backend/app/services/ontology_access.py
+++ b/backend/app/services/ontology_access.py
@@ -21,6 +21,8 @@
from sqlalchemy.orm import Session
from app.models.ontology_access import OntologyProjectAccessGrant
+from app.models.ontology import OntologyProject
+from app.models.user import User
from app.services.auth_service import get_user_by_id
from app.services.governance_audit import enqueue_audit
@@ -78,7 +80,7 @@ def _enqueue_audit(db: Session, *, security_domain_id: str, correlation_id: str,
def current_grant(db: Session, ontology_id: str, user_id: str) -> dict | None:
row = db.execute(
sa.text(
- "SELECT id, ontology_id, user_id, capabilities, revision, status, created_by, "
+ "SELECT id, ontology_id, user_id, security_domain_id, capabilities, revision, status, created_by, "
"valid_from, valid_until, revoked_by, created_at, updated_at, revoked_at "
"FROM ontology_project_access_grants "
"WHERE ontology_id = :o AND user_id = :u AND status = 'active'"
@@ -226,10 +228,10 @@ def backfill_creator_grants(db: Session) -> dict:
def _require_actor(db: Session, actor_id: str, ontology_id: str) -> dict:
- grant = current_grant(db, ontology_id, actor_id)
- if grant is None or "edit" not in grant["capabilities"]:
- raise GrantDenied("actor lacks edit on this ontology")
- return grant
+ actor = get_user_by_id(db, actor_id)
+ if actor is None:
+ raise GrantDenied("actor required")
+ return require_project_grant(db, actor, ontology_id, "edit")
def _check_ceiling(role: str, capabilities: list[str]) -> None:
@@ -426,14 +428,111 @@ def recover_owner(db: Session, *, ontology_id: str, base_finding_revision: int,
def require_project_grant(db: Session, actor, ontology_id: str, capability: str) -> dict:
- if actor is None:
+ if actor is None or not getattr(actor, "id", None) or not getattr(actor, "is_active", False):
raise GrantDenied("actor required")
- grant = current_grant(db, ontology_id, actor.id)
- if grant is None or capability not in grant["capabilities"]:
+
+ if capability not in CAPABILITIES:
+ raise GrantDenied("unknown capability")
+
+ row = db.execute(
+ sa.text(
+ "SELECT g.id, g.ontology_id, g.user_id, g.security_domain_id, g.capabilities, "
+ "g.revision, g.status, g.created_by, g.valid_from, g.valid_until, g.revoked_by, "
+ "g.created_at, g.updated_at, g.revoked_at, "
+ "o.security_domain_id AS ontology_security_domain_id, "
+ "u.security_domain_id AS actor_security_domain_id, u.is_active AS actor_is_active, "
+ "u.role AS actor_role "
+ "FROM ontology_project_access_grants g "
+ "JOIN ontology_projects o ON o.id = g.ontology_id "
+ "JOIN users u ON u.id = g.user_id "
+ "WHERE g.ontology_id = :o AND g.user_id = :u"
+ ),
+ {
+ "o": ontology_id,
+ "u": actor.id,
+ },
+ ).mappings().one_or_none()
+ if row is None:
+ raise GrantDenied("grant not held")
+
+ grant = _row_dict(row)
+ now = datetime.now(timezone.utc)
+ valid_from = _as_utc(grant["valid_from"])
+ valid_until = _as_utc(grant["valid_until"])
+ actor_domain = getattr(actor, "security_domain_id", None)
+ if (
+ not row["actor_is_active"]
+ or row["actor_security_domain_id"] != actor_domain
+ or row["ontology_security_domain_id"] != actor_domain
+ or grant["security_domain_id"] != actor_domain
+ or grant["status"] != "active"
+ or (valid_from is not None and valid_from > now)
+ or (valid_until is not None and now >= valid_until)
+ or capability not in grant["capabilities"]
+ or capability not in ROLE_CEILINGS.get(row["actor_role"], ROLE_CEILINGS["viewer"])
+ ):
raise GrantDenied("grant not held")
return grant
+def _as_utc(value):
+ """Treat naive database timestamps as UTC for grant validity checks."""
+ if value is None:
+ return None
+ if value.tzinfo is None:
+ return value.replace(tzinfo=timezone.utc)
+ return value.astimezone(timezone.utc)
+
+
+def discoverable_ontology_ids(db: Session, actor):
+ """Return a SQL selectable of ontology IDs the actor may discover.
+
+ This intentionally remains a selectable so callers can apply the grant
+ predicate before counting, paginating, or aggregating project rows.
+ Direct ``read`` grants without ``discover`` remain valid for ID-based
+ reads and therefore do not appear here.
+ """
+ if actor is None or not getattr(actor, "id", None) or not getattr(actor, "is_active", False):
+ return sa.select(OntologyProject.id).where(sa.false())
+
+ capability = "discover"
+ if capability not in ROLE_CEILINGS.get(getattr(actor, "role", None), ROLE_CEILINGS["viewer"]):
+ return sa.select(OntologyProject.id).where(sa.false())
+
+ now = datetime.now(timezone.utc)
+ grant = OntologyProjectAccessGrant
+ user = User
+ ontology = OntologyProject
+ return (
+ sa.select(ontology.id)
+ .join(
+ grant,
+ sa.and_(
+ grant.ontology_id == ontology.id,
+ grant.security_domain_id == ontology.security_domain_id,
+ ),
+ )
+ .join(
+ user,
+ sa.and_(
+ user.id == grant.user_id,
+ user.security_domain_id == grant.security_domain_id,
+ ),
+ )
+ .where(
+ grant.user_id == actor.id,
+ grant.security_domain_id == getattr(actor, "security_domain_id", None),
+ user.security_domain_id == getattr(actor, "security_domain_id", None),
+ user.is_active.is_(True),
+ grant.status == "active",
+ sa.or_(grant.valid_from.is_(None), grant.valid_from <= now),
+ sa.or_(grant.valid_until.is_(None), now < grant.valid_until),
+ sa.cast(grant.capabilities, sa.Text).like('%"discover"%'),
+ )
+ .distinct()
+ )
+
+
def _json_list(values) -> str:
import json
diff --git a/backend/app/services/v2/dataset_service.py b/backend/app/services/v2/dataset_service.py
index 14431bd6..d1cab5c8 100644
--- a/backend/app/services/v2/dataset_service.py
+++ b/backend/app/services/v2/dataset_service.py
@@ -110,11 +110,11 @@ def preview(
self,
dataset_id: str,
version_no: int,
- limit: int = 100,
+ limit: int | None = 100,
*,
version_id: str | None = None,
) -> list[dict]:
- """CSV/JSON 数据预览。无需 DuckDB, 纯 Python 处理。"""
+ """CSV/JSON 数据预览。``limit=None`` 返回完整版本。"""
query = self._db.query(DatasetVersion).filter(DatasetVersion.dataset_id == dataset_id)
if version_id is not None:
query = query.filter(DatasetVersion.id == version_id)
@@ -156,7 +156,7 @@ def preview(
if i < len(headers):
row_dict[headers[i]] = val if val is not None else ""
rows_list.append(row_dict)
- if len(rows_list) >= limit:
+ if limit is not None and len(rows_list) >= limit:
break
wb.close()
return rows_list
@@ -168,7 +168,7 @@ def preview(
reader = csv.DictReader(io.StringIO(text))
rows = []
for i, row in enumerate(reader):
- if i >= limit:
+ if limit is not None and i >= limit:
break
rows.append(dict(row))
return rows
diff --git a/backend/app/services/v2/graph/graph_analytics.py b/backend/app/services/v2/graph/graph_analytics.py
index 76109c24..09634f97 100644
--- a/backend/app/services/v2/graph/graph_analytics.py
+++ b/backend/app/services/v2/graph/graph_analytics.py
@@ -20,6 +20,7 @@ def get_neighbors(self, ontology_id: str, node_id: str, depth: int = 1) -> dict:
query = f"""
MATCH path = (n)-[r*1..{depth}]-(m)
WHERE elementId(n) = $node_id AND n.ontology_id = $ontology_id
+ AND all(node IN nodes(path) WHERE node.ontology_id = $ontology_id)
RETURN n, relationships(path) AS rels, nodes(path) AS all_nodes
LIMIT 100
"""
@@ -56,6 +57,7 @@ def shortest_path(self, ontology_id: str, src_id: str, tgt_id: str) -> dict:
WHERE elementId(s) = $src AND elementId(t) = $tgt
AND s.ontology_id = $ontology_id AND t.ontology_id = $ontology_id
MATCH p = shortestPath((s)-[*]-(t))
+ WHERE all(node IN nodes(p) WHERE node.ontology_id = $ontology_id)
RETURN [n IN nodes(p) | {id: elementId(n), labels: labels(n), name: n.name_cn}] AS path_nodes,
length(p) AS path_length
"""
@@ -80,8 +82,10 @@ def node_degree(self, ontology_id: str, node_id: str) -> dict:
query = """
MATCH (n)
WHERE elementId(n) = $node_id AND n.ontology_id = $ontology_id
- OPTIONAL MATCH (n)-[out]->()
- OPTIONAL MATCH ()-[in]->(n)
+ OPTIONAL MATCH (n)-[out]->(out_neighbor)
+ WHERE out_neighbor.ontology_id = $ontology_id
+ OPTIONAL MATCH (in_neighbor)-[in]->(n)
+ WHERE in_neighbor.ontology_id = $ontology_id
RETURN count(DISTINCT out) AS out_degree, count(DISTINCT in) AS in_degree
"""
try:
@@ -102,8 +106,8 @@ def top_connected_nodes(self, ontology_id: str, limit: int = 10) -> list[dict]:
if not self._neo4j.available:
return []
query = """
- MATCH (n)-[r]-()
- WHERE n.ontology_id = $ontology_id
+ MATCH (n)-[r]-(neighbor)
+ WHERE n.ontology_id = $ontology_id AND neighbor.ontology_id = $ontology_id
RETURN elementId(n) AS node_id,
labels(n) AS labels,
n.name_cn AS name,
diff --git a/backend/app/services/v2/incremental/operations.py b/backend/app/services/v2/incremental/operations.py
index 69e6abbb..cc86601c 100644
--- a/backend/app/services/v2/incremental/operations.py
+++ b/backend/app/services/v2/incremental/operations.py
@@ -17,6 +17,7 @@
from sqlalchemy.orm import Session
from app.models.v2.connection import Connection
+from app.models.v2.dataset import Dataset
from app.models.v2.pipeline import PipelineRunInput
from app.models.v2.refresh import (
RefreshDeadLetter,
@@ -39,12 +40,14 @@
collect_refresh_operability,
)
from app.services.v2.scheduler.schedule_service import (
- DEFAULT_RESOURCE,
ScheduleRequest,
upsert_refresh_schedule,
)
+_LEGACY_RESOURCE_SENTINEL = "__scheduled__"
+
+
class RefreshRunView(BaseModel):
"""Non-secret, serializable view of a durable refresh run."""
@@ -147,25 +150,76 @@ def _source_state(
).scalars().first()
-def _source_resource(
- db: Session, *, source_id: str, resource: str | None = None,
-) -> str:
- if resource:
- return resource
- state = _source_state(db, source_id=source_id)
- if state is not None:
- return state.resource
+def resolve_refresh_resource(db: Session, *, source_id: str) -> str:
+ """Resolve one concrete resource owned by a saved connection.
+
+ The resolver deliberately evaluates one precedence level at a time. A
+ later source of truth cannot hide an ambiguity in an earlier one, and the
+ historical ``__scheduled__`` placeholder is never a candidate.
+ """
connection = db.get(Connection, source_id)
if connection is None:
raise RefreshError("SOURCE_NOT_FOUND", f"no source connection {source_id}")
+
+ def candidates(values: list[object]) -> list[str]:
+ return sorted({str(value).strip() for value in values if value is not None and str(value).strip()})
+
# Fail closed: an `_encrypted` config that cannot be decrypted must
- # never be silently treated as "no resource hint configured" -- that
- # would resolve to DEFAULT_RESOURCE and let a manual refresh trigger
- # create a RefreshSourceState/RefreshRun for the wrong resource instead
- # of surfacing the broken configuration. Credentials are never
- # accepted from this request path; only the resource hint is read.
+ # never be silently treated as "no resource hint configured". Credentials
+ # are never accepted from this request path; only the resource hint is
+ # read.
raw = _connection_configuration(connection)
- return str(raw.get("resource") or raw.get("table") or DEFAULT_RESOURCE)
+ configured = candidates([raw.get("resource"), raw.get("table")])
+ configured = [value for value in configured if value != _LEGACY_RESOURCE_SENTINEL]
+ if len(configured) > 1:
+ raise RefreshError(
+ "REFRESH_RESOURCE_AMBIGUOUS",
+ f"connection {source_id} has multiple configured resources",
+ )
+ if configured:
+ return configured[0]
+
+ state_resources = candidates(db.execute(
+ select(RefreshSourceState.resource).where(
+ RefreshSourceState.source_id == source_id,
+ RefreshSourceState.resource != _LEGACY_RESOURCE_SENTINEL,
+ ).distinct()
+ ).scalars().all())
+ if len(state_resources) > 1:
+ raise RefreshError(
+ "REFRESH_RESOURCE_AMBIGUOUS",
+ f"connection {source_id} has multiple refresh resource states",
+ )
+ if state_resources:
+ return state_resources[0]
+
+ dataset_resources = candidates(db.execute(
+ select(Dataset.name).where(
+ Dataset.source_connection_id == source_id,
+ Dataset.name != _LEGACY_RESOURCE_SENTINEL,
+ ).distinct()
+ ).scalars().all())
+ if len(dataset_resources) > 1:
+ raise RefreshError(
+ "REFRESH_RESOURCE_AMBIGUOUS",
+ f"connection {source_id} has multiple source datasets",
+ )
+ if dataset_resources:
+ return dataset_resources[0]
+
+ raise RefreshError(
+ "REFRESH_RESOURCE_REQUIRED",
+ f"connection {source_id} has no concrete refresh resource",
+ )
+
+
+def _source_resource(
+ db: Session, *, source_id: str, resource: str | None = None,
+) -> str:
+ """Backwards-compatible explicit-resource adapter for API callers."""
+ if resource:
+ return resource
+ return resolve_refresh_resource(db, source_id=source_id)
def _connection_configuration(connection: Connection) -> dict[str, Any]:
@@ -624,6 +678,7 @@ def get_refresh_health(
__all__ = [
"RefreshRunView", "RefreshStatus", "RefreshScheduleView", "get_refresh_schedule",
+ "resolve_refresh_resource",
"trigger_refresh", "cancel_refresh_run", "replay_refresh_run",
"get_last_successful_refresh_run", "get_refresh_status", "get_refresh_health",
"schedule_view", "update_refresh_schedule",
diff --git a/backend/app/services/v2/incremental/polling.py b/backend/app/services/v2/incremental/polling.py
index 1420936c..e060cfd2 100644
--- a/backend/app/services/v2/incremental/polling.py
+++ b/backend/app/services/v2/incremental/polling.py
@@ -43,6 +43,7 @@
)
from app.services.connection.base import DeltaPage, records_to_delta_page
from app.services.connection.registry import get_connector
+from app.services.connection.config import build_connector_config, decrypt_connection_config
from app.services.v2.incremental.contract import (
_cursor_to_json,
_lock_or_create_source_state,
@@ -128,22 +129,16 @@ def _connector_for_source(
def _source_configuration(state: RefreshSourceState | None, connection: Connection | None) -> dict:
- raw = dict(connection.config or {}) if connection is not None else {}
- encrypted = raw.get("_encrypted")
- if encrypted:
- try:
- from app.services import encryption_service
-
- decrypted = encryption_service.decrypt(encrypted)
- raw = json.loads(decrypted)
- except Exception:
- logger.warning("unable to decrypt refresh source configuration", exc_info=True)
+ raw = decrypt_connection_config(connection.config if connection is not None else {})
config = dict(raw)
if state is not None and state.configuration:
# Refresh revisions carry source-owned cursor/policy settings while
# credentials and connection details remain on Connection.
config.update(state.configuration)
- return config
+ resource = state.resource if state is not None else config.get("resource")
+ if connection is None:
+ return config
+ return build_connector_config(connection.kind, config, resource=resource)
def _as_utc(value: datetime | None) -> datetime | None:
diff --git a/backend/app/services/v2/scheduler/cron_service.py b/backend/app/services/v2/scheduler/cron_service.py
index 25e5443b..94f3fdb6 100644
--- a/backend/app/services/v2/scheduler/cron_service.py
+++ b/backend/app/services/v2/scheduler/cron_service.py
@@ -72,27 +72,10 @@ def schedule_connection_sync(self, connection_id: str, cron_expr: str, *, db: "S
def schedule_pipeline_run(self, pipeline_id: str, cron_expr: str, *, db: "Session | None" = None,
**schedule_kwargs) -> dict:
- """为 Pipeline 注册定时运行任务。语义同 schedule_connection_sync。"""
- cron_params = self.parse_cron(cron_expr)
- if db is None:
- logger.info(f"Pipeline {pipeline_id} cron 校验通过(未持久化): {cron_expr}")
- return {
- "pipeline_id": pipeline_id,
- "cron": cron_expr,
- "celery_crontab": cron_params,
- "status": "validated",
- }
-
- schedule = self._persist_schedule(db, target_type="pipeline", target_id=pipeline_id,
- cron_expr=cron_expr, **schedule_kwargs)
- logger.info(f"Pipeline {pipeline_id} 调度已持久化: {cron_expr}")
- return {
- "pipeline_id": pipeline_id,
- "cron": cron_expr,
- "celery_crontab": cron_params,
- "status": "scheduled",
- "next_due_at": schedule.next_due_at.isoformat() if schedule.next_due_at else None,
- }
+ """Reject unsupported pipeline-target refresh schedules."""
+ raise ValueError(
+ "PIPELINE_SCHEDULE_UNSUPPORTED: pipeline-target schedules are not supported"
+ )
@staticmethod
def _persist_schedule(db: "Session", *, target_type: str, target_id: str, cron_expr: str,
diff --git a/backend/app/services/v2/scheduler/schedule_service.py b/backend/app/services/v2/scheduler/schedule_service.py
index f868e917..2e66b959 100644
--- a/backend/app/services/v2/scheduler/schedule_service.py
+++ b/backend/app/services/v2/scheduler/schedule_service.py
@@ -20,6 +20,7 @@
from __future__ import annotations
import uuid
+import logging
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Callable, Mapping, Sequence
@@ -38,6 +39,7 @@
)
_cron_service = CronService()
+logger = logging.getLogger(__name__)
# ScheduleRequest.target_type is the API-facing vocabulary ("connection" /
# "pipeline"); RefreshSchedule.target_type persists the same vocabulary the
@@ -47,11 +49,12 @@
_STORED_TARGET_TYPE = {"connection": "source", "pipeline": "pipeline"}
_TASK_NAME_BY_STORED_TARGET_TYPE = {"source": "refresh.connection", "pipeline": "refresh.pipeline"}
-# A connection/pipeline's schedule does not yet select a specific resource
-# (multi-resource fan-out is a later task's concern); scheduled runs are
-# scoped to this fixed placeholder resource until then.
+# Retained as an import-compatible fixture value for older callers. New
+# schedule dispatch must resolve a concrete resource before creating a run.
DEFAULT_RESOURCE = "__scheduled__"
+PIPELINE_SCHEDULE_UNSUPPORTED = "PIPELINE_SCHEDULE_UNSUPPORTED"
+
_MAX_RETRY_ATTEMPTS = 20
_DEFAULT_MAX_ATTEMPTS = 3
_MAX_RETRY_BACKOFF_SECONDS = 3600
@@ -131,6 +134,10 @@ def _validate_retry_policy(retry_policy: Mapping[str, object] | None) -> None:
def _validate_request(request: ScheduleRequest) -> ZoneInfo:
if request.target_type not in _STORED_TARGET_TYPE:
raise ValueError(f"unsupported target_type: {request.target_type!r}")
+ if request.target_type == "pipeline":
+ raise ValueError(
+ f"{PIPELINE_SCHEDULE_UNSUPPORTED}: pipeline-target schedules are not supported"
+ )
if not request.target_id:
raise ValueError("target_id is required")
if not _cron_service.validate_cron(request.cron_expr):
@@ -315,9 +322,34 @@ def _dispatch_one_schedule(
db.commit()
return None
+ if schedule.target_type == "pipeline":
+ # Rows written before pipeline-target scheduling was closed remain
+ # readable, but must never be turned into a polling run whose
+ # ``source_id`` is actually a pipeline id.
+ schedule.enabled = False
+ schedule.next_due_at = None
+ logger.warning(
+ "Skipping unsupported pipeline refresh schedule %s (%s): %s",
+ schedule.id, schedule.target_id, PIPELINE_SCHEDULE_UNSUPPORTED,
+ )
+ db.commit()
+ return None
+
source_id = schedule.target_id
- resource = DEFAULT_RESOURCE
- task_name = _TASK_NAME_BY_STORED_TARGET_TYPE[schedule.target_type]
+ from app.services.v2.incremental.operations import resolve_refresh_resource
+ from app.schemas.refresh import RefreshError
+
+ try:
+ resource = resolve_refresh_resource(db, source_id=source_id)
+ except RefreshError as exc:
+ logger.warning(
+ "Skipping refresh schedule %s for source %s: %s",
+ schedule.id, source_id, exc.reason_code,
+ )
+ schedule.next_due_at = _next_fire_utc(schedule.cron_expression, tz, max(due_at, now))
+ db.commit()
+ return None
+ task_name = _TASK_NAME_BY_STORED_TARGET_TYPE["source"]
idempotency_key = f"schedule:{schedule.id}:{due_at.isoformat()}"
run = _lock_schedule_occurrence(
db, source_id=source_id, resource=resource, idempotency_key=idempotency_key,
diff --git a/backend/app/tasks/extraction.py b/backend/app/tasks/extraction.py
index fa4c21ae..35a8f52e 100644
--- a/backend/app/tasks/extraction.py
+++ b/backend/app/tasks/extraction.py
@@ -1,6 +1,7 @@
from app.tasks.celery_app import celery_app
import logging
+import re
logger = logging.getLogger(__name__)
@@ -8,6 +9,7 @@
from app.models import ( # noqa: E402, F401
user, ontology, file, prompt, model_config,
entity, logic as logic_model, action, relation, extraction_task, rules_config,
+ entity_instance, entity_instance_relation,
)
# ── 概念/实例分离指令 — 追加到所有提取 Prompt 之后 ───────────────────────────
@@ -20,7 +22,20 @@
不要为文档中提到的每个具体命名对象(如具体公司名、具体人名、具体订单号、具体条款标题)单独创建 entity。
文档中提到的具体命名实例,放入新增的 instances 数组,通过 entity_type 字段关联到其所属概念的 name_cn:
"instances": [{"entity_type": "所属概念的name_cn", "name_cn": "实例名称", "name_en": "可选", "properties": {"属性1": "值"}, "confidence": 0.9}]
-relations 只在概念之间建立(如 供应商 -[supply]-> 产品),不要在具体实例之间建立关系。"""
+relations 只在概念之间建立(如 供应商 -[supply]-> 产品),不要在具体实例之间建立关系。
+
+如果两个具体命名实例之间存在文档明确写出的因果/治疗/相互作用等关系(如"2型糖尿病 导致 糖尿病肾病""阿司匹林 与 华法林 相互作用"),
+放入新增的 instance_relations 数组,不要因为"relations 只在概念之间建立"而丢弃这类信息:
+"instance_relations": [{"source": "实例名称", "target": "实例名称", "type": "causes|interacts_with|治疗|等英文或语义明确的关系类型", "confidence": 0.9}]
+source/target 必须是 instances 数组里已出现的 name_cn,不要引用未提取的实例或概念实体本身。
+
+属性的枚举值/分级/分档/阶段标签(如信用分层的"A层""B层""C层"、逾期阶段的"M0""M1""M2+"、
+融资轮次的"A轮融资""D轮融资")不是独立的概念,也不是需要单独建 instance 的命名对象——它们是所属
+概念(如"信用分层""逾期阶段""融资")某个属性的可选取值。不要为它们创建 entity,也不要创建
+instance;只需在所属概念的 properties 里以枚举值形式体现(如信用分层.properties.levels = ["A层","B层","C层"])。
+
+业务规则、政策、判断条件、定价/审批/校验类规则(如"促销定价规则""审批超时规则""授信额度校验")
+不是实体,禁止作为 entity 或 instance 输出,必须放入 logic_rules 数组。"""
# ── Extraction results merge (方案A) ───────────────────────────────────────
@@ -31,6 +46,7 @@ def _merge_extraction_results(results: list[dict]) -> dict:
all_logic: list[dict] = []
all_actions: list[dict] = []
all_instances: list[dict] = []
+ all_instance_relations: list[dict] = []
# 收集全部
for r in results:
@@ -53,6 +69,9 @@ def _merge_extraction_results(results: list[dict]) -> dict:
for inst in (r.get("instances") or []):
if isinstance(inst, dict):
all_instances.append(inst)
+ for ir in (r.get("instance_relations") or []):
+ if isinstance(ir, dict):
+ all_instance_relations.append(ir)
# 实体去重(按 name_cn,保留属性最丰富的)
seen_entities: dict[str, dict] = {}
@@ -116,12 +135,22 @@ def _merge_extraction_results(results: list[dict]) -> dict:
seen_instances.add(key)
instances.append(inst)
+ # 实例关系去重(source-target-type 唯一)
+ seen_inst_rels: set = set()
+ instance_relations = []
+ for ir in all_instance_relations:
+ key = (ir.get("source", ""), ir.get("target", ""), ir.get("type", ""))
+ if key not in seen_inst_rels and key[0] and key[1]:
+ seen_inst_rels.add(key)
+ instance_relations.append(ir)
+
return {
"entities": entities,
"relations": relations,
"logic_rules": list(seen_logic.values()),
"actions": list(seen_actions.values()),
"instances": instances,
+ "instance_relations": instance_relations,
}
@@ -241,6 +270,75 @@ def _fuzzy_resolve_entity(name: str, name_to_id: dict) -> str | None:
return candidates[0][1]
+# ── Tiered model selection (graph-engineering playbook, model selection) ────
+# Extraction is high-volume and schema-constrained — speed/cost dominate, so
+# it always uses whatever model the user picked. Resolution and relation
+# inference weigh conflicting evidence — reasoning quality dominates — so
+# when the user picked an explicitly fast/cheap variant, we look for a
+# stronger sibling already configured on the same model config and use that
+# instead. No new user-facing setting: this reads only what's already in
+# ModelConfig.models.
+_FAST_MODEL_HINTS = ("flash", "mini", "haiku", "lite", "turbo", "nano", "fast")
+
+
+def _select_reasoning_model(model_name: str, available_models: list) -> str:
+ """Pick the model to use for judgment-heavy stages (resolution, relation
+ inference). Falls back to `model_name` unchanged unless the user's pick
+ looks like a fast/cheap variant AND a non-fast sibling exists in the same
+ model config — so behavior is identical to today whenever no better
+ candidate is configured."""
+ if not any(hint in model_name.lower() for hint in _FAST_MODEL_HINTS):
+ return model_name
+ for candidate in available_models or []:
+ if isinstance(candidate, str) and candidate != model_name and not any(
+ hint in candidate.lower() for hint in _FAST_MODEL_HINTS
+ ):
+ return candidate
+ return model_name
+
+
+# ── Long-document chunking (graph-engineering playbook, scaling guidance) ───
+_CHUNK_MAX_CHARS = 12000
+_CHUNK_OVERLAP_CHARS = 500
+_SECTION_HEADER = re.compile(r"^#{1,3}\s+.*$", re.MULTILINE)
+
+
+def _split_sections(md: str) -> list[str]:
+ """Split markdown into section chunks at heading boundaries. Falls back to
+ paragraph boundaries when the document has no headings at all."""
+ starts = [m.start() for m in _SECTION_HEADER.finditer(md)]
+ if not starts or starts[0] != 0:
+ starts = [0] + starts
+ if len(starts) > 1:
+ return [md[s:e] for s, e in zip(starts, starts[1:] + [len(md)]) if md[s:e].strip()]
+ return [p for p in md.split("\n\n") if p.strip()]
+
+
+def _chunk_document(md: str, max_chars: int = _CHUNK_MAX_CHARS, overlap_chars: int = _CHUNK_OVERLAP_CHARS) -> list[str]:
+ """Chunk a long document at section boundaries so a single extraction call
+ never has to hold the whole thing — entities/relations still land within
+ the same chunk as their context. Each chunk after the first repeats the
+ tail of the previous chunk as overlap, so a concept mentioned at a section
+ boundary isn't split from the relations described just after it. Short
+ documents (the common case) pass through unchanged as a single chunk."""
+ if len(md) <= max_chars:
+ return [md]
+
+ sections = _split_sections(md)
+ chunks: list[str] = []
+ current = ""
+ for section in sections:
+ if current and len(current) + len(section) > max_chars:
+ chunks.append(current)
+ overlap = current[-overlap_chars:] if overlap_chars > 0 else ""
+ current = overlap + section
+ else:
+ current += section
+ if current.strip():
+ chunks.append(current)
+ return chunks or [md]
+
+
@celery_app.task(bind=True)
def run_extraction(self, task_id: str):
import app.models # noqa: F401 — register all tables for FK resolution
@@ -286,6 +384,12 @@ def run_extraction(self, task_id: str):
task.status = "failed"; task.error = "Model or prompt not found"; db.commit(); return
model_name = task.parameters.get("model_name", "")
+ # resolution/relation-inference weigh conflicting evidence — use a
+ # stronger sibling model already configured on this model config when
+ # the user picked a fast/cheap variant for extraction (see
+ # _select_reasoning_model); falls back to model_name unchanged
+ # otherwise, so this is a no-op unless a better candidate exists
+ reasoning_model_name = _select_reasoning_model(model_name, model_cfg.models)
config_dict = {
"provider": model_cfg.provider,
"api_key": decrypt(model_cfg.api_key_encrypted or ""),
@@ -309,14 +413,19 @@ def _clean(text: str) -> str:
max_workers = 1 # serial extraction to avoid OOM with large LLM payloads
completed = 0
- task.progress = {"stage": f"extracting files 0/{len(valid_mds)} (parallel ×{max_workers})", "pct": 20}
+ # a long document is chunked at section boundaries first (see
+ # _chunk_document) — most documents are short and pass through as a
+ # single chunk, so this is a no-op for today's typical corpus
+ units = [(f, chunk) for f in valid_mds for chunk in _chunk_document(_clean(f.converted_md or ""))]
+ total_units = len(units)
+
+ task.progress = {"stage": f"extracting files 0/{total_units} (parallel ×{max_workers})", "pct": 20}
db.commit()
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = {}
- for f in valid_mds:
- md = _clean(f.converted_md or "")
- futures[executor.submit(extract_ontology, md, prompt_content, config_dict, model_name)] = f
+ for f, chunk in units:
+ futures[executor.submit(extract_ontology, chunk, prompt_content, config_dict, model_name)] = f
for future in as_completed(futures):
f = futures[future]
@@ -327,7 +436,7 @@ def _clean(text: str) -> str:
except Exception:
logger.exception("extract_ontology failed for file %s", getattr(f, "id", f))
completed += 1
- task.progress = {"stage": f"extracting files {completed}/{len(valid_mds)} (parallel ×{max_workers})", "pct": 20 + 35 * completed // len(valid_mds)}
+ task.progress = {"stage": f"extracting files {completed}/{total_units} (parallel ×{max_workers})", "pct": 20 + 35 * completed // total_units}
db.commit()
if all_results:
@@ -337,11 +446,23 @@ def _clean(text: str) -> str:
# so a concept extracted under different surface forms — whether
# across files or within one document — would otherwise fracture
# into separate nodes
- if len(result.get("entities") or []) > 1:
+ new_entity_count = len(result.get("entities") or [])
+ # incremental-update guidance: resolve this run's new entities
+ # against the ontology's already-published entities too, so a
+ # re-run doesn't create a duplicate for a surface-form variant of
+ # something that already exists
+ existing_entities_for_resolution = [
+ {"name_cn": e.name_cn, "description": e.description, "type": e.type}
+ for e in db.query(Entity).filter(Entity.ontology_id == task.ontology_id).all()
+ ]
+ if new_entity_count > 1 or (new_entity_count == 1 and existing_entities_for_resolution):
task.progress = {"stage": "resolving entities", "pct": 58}
db.commit()
from app.services.llm_service import resolve_entities, apply_entity_resolution
- alias_map = resolve_entities(result["entities"], config_dict, model_name)
+ alias_map = resolve_entities(
+ result["entities"], config_dict, reasoning_model_name,
+ existing_entities=existing_entities_for_resolution,
+ )
result = apply_entity_resolution(result, alias_map)
else:
# 全部失败时回退到合并文本单次提取
@@ -390,7 +511,7 @@ def _clean(text: str) -> str:
db.commit()
extra_rels = infer_relations(
entities_extracted, relations_extracted,
- combined_text, config_dict, model_name
+ combined_text, config_dict, reasoning_model_name
)
if extra_rels:
# Accept relations where both endpoints fuzzy-match a known entity name
@@ -439,7 +560,10 @@ def _clean(text: str) -> str:
ent = existing_ent_map[name_cn]
# Always allow LLM to enrich description, properties, name_en, name_abbr
if e_data.get("description"): ent.description = e_data["description"]
- if props: ent.properties = props
+ # merge, don't replace — a prior run's hub summary
+ # (summary/key_facts/summary_degree) must survive a re-run
+ # that doesn't touch this entity's properties otherwise
+ if props: ent.properties = {**(ent.properties or {}), **props}
if e_data.get("name_en"): ent.name_en = e_data["name_en"]
if name_abbr: ent.name_abbr = name_abbr
if e_data.get("type"): ent.type = e_data["type"]
@@ -472,6 +596,10 @@ def _clean(text: str) -> str:
# ── 写入实例数据(EntityInstance 表)— 挂在概念实体下,对齐 Pipeline Mapping ──
from app.models.entity_instance import EntityInstance
import hashlib as _hl
+ # instance_relations (below) needs to resolve a source/target instance
+ # name back to its stable id and owning concept entity id
+ instance_name_to_id: dict = {}
+ instance_id_to_entity_id: dict = {}
for inst_data in result.get("instances", []):
if not isinstance(inst_data, dict):
continue
@@ -506,11 +634,14 @@ def _clean(text: str) -> str:
id=stable_id, entity_id=concept_id, ontology_id=task.ontology_id,
row_identity=inst_name, row_data=inst_props,
))
+ instance_name_to_id[inst_name] = stable_id
+ instance_id_to_entity_id[stable_id] = concept_id
db.commit()
# ── Fix 2+4: upsert relations (by source_id, target_id, type) ────────
existing_rels = db.query(Relation).filter(Relation.ontology_id == task.ontology_id).all()
existing_rel_set = {(r.source_entity, r.target_entity, r.type) for r in existing_rels}
+ relation_id_by_key: dict = {(r.source_entity, r.target_entity, r.type): r.id for r in existing_rels}
for rel in result.get("relations", []):
src_name = rel.get("source") or rel.get("source_entity", "")
@@ -522,12 +653,70 @@ def _clean(text: str) -> str:
if rel_type in ("关联", "未知", "相关", "其他", "") or not rel_type.isascii():
continue
if src_id and tgt_id and (src_id, tgt_id, rel_type) not in existing_rel_set:
+ new_rel_id = str(uuid.uuid4())
db.add(Relation(
- id=str(uuid.uuid4()), ontology_id=task.ontology_id,
+ id=new_rel_id, ontology_id=task.ontology_id,
source_entity=src_id, target_entity=tgt_id,
type=rel_type, confidence=rel.get("confidence", 0.85),
))
existing_rel_set.add((src_id, tgt_id, rel_type))
+ relation_id_by_key[(src_id, tgt_id, rel_type)] = new_rel_id
+
+ # instance_relations below may reuse any of the relations just added
+ # via relation_id_by_key as their relation_definition_id FK — flush
+ # now so those rows exist before EntityInstanceRelation references them
+ db.flush()
+
+ # ── instance-level relations (graph-engineering playbook): specific
+ # named things (e.g. 2型糖尿病, 阿司匹林) keep their own relation graph
+ # via EntityInstanceRelation, anchored to the concept-level Relation
+ # it instantiates (auto-created here if the LLM didn't also emit it) ──
+ from app.models.entity_instance_relation import EntityInstanceRelation
+ existing_inst_rels = db.query(EntityInstanceRelation).filter(
+ EntityInstanceRelation.ontology_id == task.ontology_id).all()
+ existing_inst_rel_set = {
+ (r.source_instance_id, r.target_instance_id, r.relation_definition_id)
+ for r in existing_inst_rels
+ }
+
+ for ir in result.get("instance_relations", []):
+ if not isinstance(ir, dict):
+ continue
+ src_iid = _fuzzy_resolve_entity(ir.get("source", ""), instance_name_to_id)
+ tgt_iid = _fuzzy_resolve_entity(ir.get("target", ""), instance_name_to_id)
+ if not src_iid or not tgt_iid:
+ continue
+ ir_type = (ir.get("type") or "RELATED").strip()
+ if ir_type in ("关联", "未知", "相关", "其他", "") or not ir_type.isascii():
+ continue
+ src_concept_id = instance_id_to_entity_id.get(src_iid)
+ tgt_concept_id = instance_id_to_entity_id.get(tgt_iid)
+ if not src_concept_id or not tgt_concept_id:
+ continue
+ concept_key = (src_concept_id, tgt_concept_id, ir_type)
+ rel_def_id = relation_id_by_key.get(concept_key)
+ if not rel_def_id:
+ rel_def_id = str(uuid.uuid4())
+ db.add(Relation(
+ id=rel_def_id, ontology_id=task.ontology_id,
+ source_entity=src_concept_id, target_entity=tgt_concept_id,
+ type=ir_type, confidence=ir.get("confidence", 0.8),
+ ))
+ existing_rel_set.add(concept_key)
+ relation_id_by_key[concept_key] = rel_def_id
+ # the FK from entity_instance_relations to relations must see
+ # this row on insert — flush now rather than trust autoflush
+ # ordering across the two tables
+ db.flush()
+ edge_key = (src_iid, tgt_iid, rel_def_id)
+ if edge_key not in existing_inst_rel_set:
+ db.add(EntityInstanceRelation(
+ id=str(uuid.uuid4()), ontology_id=task.ontology_id,
+ source_instance_id=src_iid, target_instance_id=tgt_iid,
+ relation_definition_id=rel_def_id,
+ properties={"confidence": ir.get("confidence", 0.8)},
+ ))
+ existing_inst_rel_set.add(edge_key)
# ── Keyword matching helpers (unchanged) ─────────────────────────────
all_entity_names = [
@@ -584,6 +773,12 @@ def _match_logic_rules(text: str, logic_name_to_id: dict) -> list:
if not llm_linked:
combined = " ".join(filter(None, [name_cn, r_data.get("definition",""), r_data.get("description","")]))
llm_linked = _match_entities(combined, all_entity_names)
+ # A rule-category label (e.g. "反欺诈规则") is not a concept entity —
+ # it belongs in logic_rules, not entities — so a linked_entities
+ # reference to one is a dangling reference, same as a relation
+ # pointing at an entity the model never extracted; drop it rather
+ # than let it surface as a broken-reference validation warning.
+ llm_linked = [n for n in llm_linked if n in entity_name_to_id]
if name_cn in existing_rule_map:
rule = existing_rule_map[name_cn]
@@ -623,6 +818,9 @@ def _match_logic_rules(text: str, logic_name_to_id: dict) -> list:
if not linked_ents:
combined = " ".join(filter(None, [name_cn, a_data.get("description","")]))
linked_ents = _match_entities(combined, all_entity_names)
+ # Same dangling-reference cleanup as logic_rules above — a
+ # rule-category label is not a concept entity.
+ linked_ents = [n for n in linked_ents if n in entity_name_to_id]
linked_logic_names = a_data.get("linked_logic_names", [])
linked_ids = [logic_name_to_id[n] for n in linked_logic_names if n in logic_name_to_id]
@@ -661,6 +859,50 @@ def _match_logic_rules(text: str, logic_name_to_id: dict) -> list:
if project:
project.status = "created"
+ # graph-engineering playbook diagnostics: computed over the ontology's
+ # full saved graph (not just this run's output), so a stale connected-
+ # components count from a prior run never masks today's actual state.
+ # session autoflush is off (app/database.py), so this run's own
+ # entity/relation adds above must be flushed before querying them back.
+ db.flush()
+ from app.services.graph_diagnostics import compute_graph_diagnostics
+ final_entities = db.query(Entity).filter(Entity.ontology_id == task.ontology_id).all()
+ final_relations = db.query(Relation).filter(Relation.ontology_id == task.ontology_id).all()
+ graph_diagnostics = compute_graph_diagnostics(
+ [{"id": e.id, "name_cn": e.name_cn} for e in final_entities],
+ [(r.source_entity, r.target_entity) for r in final_relations],
+ )
+ task.validation_report = {**(task.validation_report or {}), "graph_diagnostics": graph_diagnostics}
+
+ # hub-node summarization (playbook Section V.A): only entities with
+ # enough graph connectivity to benefit from cross-relation synthesis;
+ # re-summarizes only when degree changed since the last summary, so a
+ # re-run doesn't re-call the model for an unchanged hub every time
+ _HUB_DEGREE_THRESHOLD = 3
+ degree_by_id: dict = {e.id: 0 for e in final_entities}
+ relation_lines_by_id: dict = {}
+ name_by_id = {e.id: e.name_cn for e in final_entities}
+ for r in final_relations:
+ if r.source_entity in degree_by_id:
+ degree_by_id[r.source_entity] += 1
+ if r.target_entity in degree_by_id:
+ degree_by_id[r.target_entity] += 1
+ line = f"({name_by_id.get(r.source_entity, r.source_entity)}) --[{r.type}]--> ({name_by_id.get(r.target_entity, r.target_entity)})"
+ relation_lines_by_id.setdefault(r.source_entity, []).append(line)
+ relation_lines_by_id.setdefault(r.target_entity, []).append(line)
+
+ from app.services.llm_service import summarize_entity
+ for e in final_entities:
+ degree = degree_by_id.get(e.id, 0)
+ if degree < _HUB_DEGREE_THRESHOLD:
+ continue
+ if (e.properties or {}).get("summary_degree") == degree:
+ continue # graph neighborhood unchanged since last summary
+ relations_text = "\n".join(sorted(set(relation_lines_by_id.get(e.id, []))))
+ profile = summarize_entity(e.name_cn, e.description or "", relations_text, config_dict, reasoning_model_name)
+ if profile:
+ e.properties = {**(e.properties or {}), **profile, "summary_degree": degree}
+
task.status = "completed"
task.progress = {"stage": "done", "pct": 100}
db.commit()
@@ -676,6 +918,12 @@ def _match_logic_rules(text: str, logic_name_to_id: dict) -> list:
if task:
task.status = "failed"
task.error = str(e)
+ # the router set the Ontology to "creating" when this task was
+ # queued — on failure it must go back to "draft" or it stays
+ # stuck showing "creating" forever with no task left to finish it
+ project = db.query(OntologyProject).filter(OntologyProject.id == task.ontology_id).first()
+ if project and project.status == "creating":
+ project.status = "draft"
db.commit()
except Exception:
# session 可能已损坏,用新 session 兜底标记失败,避免任务永远卡在 running
@@ -686,6 +934,9 @@ def _match_logic_rules(text: str, logic_name_to_id: dict) -> list:
if task:
task.status = "failed"
task.error = str(e)
+ project = fresh_db.query(OntologyProject).filter(OntologyProject.id == task.ontology_id).first()
+ if project and project.status == "creating":
+ project.status = "draft"
fresh_db.commit()
fresh_db.close()
except Exception:
@@ -700,6 +951,8 @@ def _sync_neo4j(db, ontology_id: str) -> None:
from app.services.v2.graph.neo4j_service import Neo4jService
from app.models.entity import Entity
from app.models.relation import Relation
+ from app.models.entity_instance import EntityInstance
+ from app.models.entity_instance_relation import EntityInstanceRelation
svc = Neo4jService()
if not svc.available:
@@ -747,6 +1000,39 @@ def _sync_neo4j(db, ontology_id: str) -> None:
props={"id": r.id, "ontology_id": ontology_id, "confidence": r.confidence or 0.85},
)
+ # 具体命名实例(如 2型糖尿病、阿司匹林)作为独立节点同步,标签在概念类型
+ # 后加 Instance 后缀,与概念节点区分开,避免图查询把概念和实例混为一谈
+ instances = db.query(EntityInstance).filter(EntityInstance.ontology_id == ontology_id).all()
+ instance_label_map: dict[str, str] = {}
+ for inst in instances:
+ concept_type = entity_type_map.get(inst.entity_id, "OntologyEntity")
+ label = f"{concept_type}Instance"
+ instance_label_map[inst.id] = label
+ props = {
+ **(inst.row_data or {}),
+ "id": inst.id,
+ "ontology_id": ontology_id,
+ "entity_id": inst.entity_id,
+ "name_cn": inst.row_identity,
+ }
+ svc.upsert_entity(label, props, key_field="id")
+
+ # 实例级关系(语义概念层面的 relation 已在上面写入;具体实例之间的边只
+ # 存在于 Neo4j,不进 Postgres relations 表)
+ inst_relations = db.query(EntityInstanceRelation).filter(
+ EntityInstanceRelation.ontology_id == ontology_id).all()
+ relation_type_by_id = {r.id: r.type for r in relations}
+ for ir in inst_relations:
+ src_label = instance_label_map.get(ir.source_instance_id, "OntologyEntityInstance")
+ tgt_label = instance_label_map.get(ir.target_instance_id, "OntologyEntityInstance")
+ rel_type = (relation_type_by_id.get(ir.relation_definition_id) or "RELATED").upper().replace(" ", "_").replace("-", "_")
+ svc.upsert_relation(
+ src_label=src_label, src_key=ir.source_instance_id,
+ tgt_label=tgt_label, tgt_key=ir.target_instance_id,
+ rel_type=rel_type,
+ props={"id": ir.id, "ontology_id": ontology_id, **(ir.properties or {})},
+ )
+
svc.close()
except Exception:
logger.warning("Neo4j sync failed for ontology %s; extraction result unaffected", ontology_id, exc_info=True)
diff --git a/backend/app/tasks/topology.py b/backend/app/tasks/topology.py
index e30f1fa3..9f6bcdef 100644
--- a/backend/app/tasks/topology.py
+++ b/backend/app/tasks/topology.py
@@ -76,6 +76,19 @@
"agent.retention_purge": {"queue": QUEUE_HOUSEKEEPING},
}
+# These four share one queue/worker pool. Without acks_late + a time limit, a
+# single hung LLM call (or a worker dying mid-task) permanently wedges one of
+# `--concurrency` slots with nothing to free it — every later task on this
+# queue then sits "queued" forever with no worker left to pick it up.
+ARTIFACT_EXTRACTION_TASK_NAMES = (
+ "app.tasks.extraction.run_extraction",
+ "app.tasks.audit.run_audit",
+ "app.tasks.v2.mapping_apply.mapping_apply_task",
+ "app.tasks.v2.pipeline_run.pipeline_run_task",
+)
+ARTIFACT_EXTRACTION_SOFT_TIME_LIMIT_SECONDS = 1800
+ARTIFACT_EXTRACTION_HARD_TIME_LIMIT_SECONDS = 1860
+
# ── Bounded refresh worker settings ─────────────────────────────────────
@dataclass(frozen=True)
@@ -227,3 +240,12 @@ def configure_celery_topology(celery_app: Celery) -> None:
}
for name in REFRESH_TASK_NAMES
}
+ celery_app.conf.task_annotations.update({
+ name: {
+ "acks_late": True,
+ "reject_on_worker_lost": True,
+ "soft_time_limit": ARTIFACT_EXTRACTION_SOFT_TIME_LIMIT_SECONDS,
+ "time_limit": ARTIFACT_EXTRACTION_HARD_TIME_LIMIT_SECONDS,
+ }
+ for name in ARTIFACT_EXTRACTION_TASK_NAMES
+ })
diff --git a/backend/app/tasks/v2/pipeline_run.py b/backend/app/tasks/v2/pipeline_run.py
index 7326181d..5574d1b6 100644
--- a/backend/app/tasks/v2/pipeline_run.py
+++ b/backend/app/tasks/v2/pipeline_run.py
@@ -3,11 +3,14 @@
from datetime import datetime, timezone
import logging
from pathlib import Path
-from typing import Sequence
+from typing import Mapping, Sequence
import uuid
logger = logging.getLogger(__name__)
+from app.services.connection.config import build_connector_config, safe_connection_error
+from app.services.connection.registry import get_connector
+
def get_celery_app():
try:
@@ -202,6 +205,7 @@ def _find_dataset_for_file(db, filename: str):
def _collect_sources(db, pl) -> list[dict]:
+ from app.models.v2.connection import Connection
from app.models.v2.dataset import Dataset
sources: list[dict] = []
@@ -209,7 +213,29 @@ def _collect_sources(db, pl) -> list[dict]:
for node in definition.get("nodes") or []:
if node.get("type") != "connector":
continue
- for file_info in (node.get("config") or {}).get("files", []) or []:
+ config = node.get("config") or {}
+ source_type = str(config.get("source_type") or "").lower()
+ if source_type in {"mysql", "postgres", "postgresql"}:
+ connection_id = str(config.get("connection_id") or "")
+ resource = str(config.get("resource") or config.get("table") or "")
+ if not connection_id:
+ raise ValueError("Database connector must select a saved connection")
+ if not resource:
+ raise ValueError("Database connector must select a table")
+ connection = db.get(Connection, connection_id)
+ if connection is None:
+ raise ValueError(f"Saved connection not found: {connection_id}")
+ sources.append({
+ "connection_id": connection.id,
+ "resource": resource,
+ "filename": resource,
+ "route": "A",
+ "kind": "structured",
+ "dataset_id": None,
+ })
+ continue
+
+ for file_info in config.get("files", []) or []:
filename = file_info.get("name") or file_info.get("filename") or ""
dataset_id = file_info.get("dataset_id")
ds = db.query(Dataset).filter(Dataset.id == dataset_id).first() if dataset_id else None
@@ -237,13 +263,146 @@ def _collect_sources(db, pl) -> list[dict]:
seen: set[str] = set()
unique_sources = []
for source in sources:
- if source["dataset_id"] in seen:
+ key = source.get("dataset_id") or (
+ f"connection:{source.get('connection_id')}:{source.get('resource')}"
+ )
+ if key in seen:
continue
- seen.add(source["dataset_id"])
+ seen.add(key)
unique_sources.append(source)
return unique_sources
+def _existing_database_dataset(db, connection_id: str, resource: str):
+ from app.models.v2.dataset import Dataset
+
+ return db.query(Dataset).filter(
+ Dataset.source_connection_id == connection_id,
+ Dataset.name == resource,
+ ).order_by(Dataset.created_at.desc()).first()
+
+
+def _pinned_database_version(db, dataset_id: str, input_dataset_version_ids) -> object | None:
+ if not input_dataset_version_ids:
+ return None
+ from app.models.v2.dataset import DatasetVersion
+
+ for version_id in input_dataset_version_ids:
+ version = db.get(DatasetVersion, version_id)
+ if version is not None and version.dataset_id == dataset_id:
+ return version
+ return None
+
+
+def _resolve_pinned_input_versions(db, sources: list[dict], input_dataset_version_ids) -> dict[int, object]:
+ """Resolve one immutable input version for every source before any pull.
+
+ A refresh-triggered pipeline must not silently fill a missing source pin
+ from a current/latest dataset version. SQL sources may not have a
+ materialized Dataset until their first unpinned run, so their candidate
+ Dataset rows are resolved by the saved connection/resource identity.
+ """
+ if not input_dataset_version_ids:
+ return {}
+
+ from app.models.v2.dataset import Dataset, DatasetVersion
+
+ requested_ids = list(input_dataset_version_ids)
+ versions = db.query(DatasetVersion).filter(DatasetVersion.id.in_(set(requested_ids))).all()
+ versions_by_id = {version.id: version for version in versions}
+ resolved: dict[int, object] = {}
+
+ for source in sources:
+ dataset_ids: set[str] = set()
+ if source.get("dataset_id"):
+ dataset_ids.add(source["dataset_id"])
+ elif source.get("connection_id"):
+ datasets = db.query(Dataset).filter(
+ Dataset.source_connection_id == source["connection_id"],
+ Dataset.name == source["resource"],
+ ).all()
+ dataset_ids.update(dataset.id for dataset in datasets)
+
+ matches = [
+ versions_by_id[version_id]
+ for version_id in requested_ids
+ if version_id in versions_by_id
+ and versions_by_id[version_id].dataset_id in dataset_ids
+ ]
+ if len(matches) != 1:
+ source_name = source.get("resource") or source.get("filename") or "unknown"
+ raise ValueError(
+ "PINNED_INPUTS_INCOMPLETE: source "
+ f"{source_name!r} requires exactly one pinned dataset version"
+ )
+ resolved[id(source)] = matches[0]
+
+ return resolved
+
+
+def _snapshot_database_source(
+ db, svc, source: dict, input_dataset_version_ids, selected_version=None,
+) -> object:
+ """Materialize a selected SQL table as an immutable DatasetVersion.
+
+ A refresh-triggered run can provide an exact pinned version; in that case
+ the live table is never read again. Manual runs without a pin read the
+ selected resource once and persist a new snapshot before Route A loads it.
+ """
+ from app.models.v2.connection import Connection
+ from app.models.v2.dataset import Dataset
+
+ connection = db.get(Connection, source["connection_id"])
+ if connection is None:
+ raise ValueError(f"Saved connection not found: {source['connection_id']}")
+ dataset = _existing_database_dataset(db, connection.id, source["resource"])
+ if selected_version is not None:
+ # A pinned SQL version is already immutable. Reuse it directly and
+ # recover the Dataset identity without touching the live connector.
+ if dataset is None:
+ dataset = db.get(Dataset, selected_version.dataset_id)
+ if (
+ dataset is None
+ or dataset.id != selected_version.dataset_id
+ or dataset.source_connection_id != connection.id
+ or dataset.name != source["resource"]
+ ):
+ raise ValueError(
+ "PINNED_INPUTS_INCOMPLETE: pinned version does not belong to "
+ f"{connection.id}/{source['resource']}"
+ )
+ source["dataset_id"] = dataset.id
+ return selected_version
+ if dataset is not None:
+ pinned = _pinned_database_version(db, dataset.id, input_dataset_version_ids)
+ if pinned is not None:
+ source["dataset_id"] = dataset.id
+ return pinned
+
+ config = build_connector_config(
+ connection.kind, connection.config, resource=source["resource"],
+ )
+ # A pipeline table selector is authoritative. Do not allow an optional
+ # saved query to redirect it to an arbitrary statement.
+ config.pop("query", None)
+ connector = get_connector(connection.kind, config)
+ rows = connector.pull_full(source["resource"])
+ if isinstance(rows, Mapping):
+ rows = [dict(rows)]
+ else:
+ rows = [dict(row) for row in (rows or [])]
+
+ if dataset is None:
+ dataset = svc.create_dataset(
+ name=source["resource"], kind="structured", connection_id=connection.id,
+ )
+ version = svc.create_version(
+ dataset.id, _safe_csv_bytes(rows), rowcount=len(rows),
+ )
+ source["dataset_id"] = dataset.id
+ return version
+
+
def _load_source_rows(
db,
svc,
@@ -273,8 +432,14 @@ def _load_source_rows(
}]
if version is None:
return []
+ # A database snapshot is already a bounded, immutable input selected by
+ # the pipeline. Loading only the preview cap would silently drop rows
+ # from the pipeline, so request the complete version for DB sources while
+ # retaining the existing preview cap for file sources.
+ preview_limit = None if source.get("connection_id") else limit
return svc.preview(
- source["dataset_id"], version.version_no, limit=limit, version_id=version.id,
+ source["dataset_id"], version.version_no,
+ limit=preview_limit, version_id=version.id,
)
@@ -375,6 +540,21 @@ def _save_curated_outputs(db, svc, pl, source: dict, data: list[dict], ctx, mult
return [_save_curated_dataset(db, svc, pl, source, data, ctx, multi_source)]
+def _set_pipeline_status_if_latest(db, pipeline, run, status: str) -> bool:
+ """Update the pipeline summary only for the newest deterministic run."""
+ from app.models.v2.pipeline import PipelineRun
+
+ latest = db.query(PipelineRun).filter(
+ PipelineRun.pipeline_id == run.pipeline_id,
+ ).order_by(
+ PipelineRun.created_at.desc(), PipelineRun.id.desc(),
+ ).first()
+ if latest is None or latest.id != run.id:
+ return False
+ pipeline.status = status
+ return True
+
+
def pipeline_run_task(
pipeline_id: str, run_id: str,
input_dataset_version_ids: Sequence[str] | None = None,
@@ -382,20 +562,18 @@ def pipeline_run_task(
"""Pipeline 执行任务 — 支持 DAG 编译 + 节点状态追踪"""
from app.database import SessionLocal
from app.models.v2.pipeline import Pipeline, PipelineRun, PipelineRunInput
- from app.models.v2.dataset import DatasetVersion
from app.services.v2.pipeline.base import PipelineContext
from app.services.v2.pipeline.dag_compiler import compile_definition
from app.services.v2.dataset_service import DatasetService
from app.services.v2.incremental.polling import select_pinned_dataset_version
db = SessionLocal()
+ run = None
+ pl = None
try:
run = db.query(PipelineRun).filter(PipelineRun.id == run_id).first()
if not run:
return
- run.status = "running"
- run.started_at = datetime.now(timezone.utc)
- db.commit()
pl = db.query(Pipeline).filter(Pipeline.id == pipeline_id).first()
if not pl:
@@ -404,6 +582,11 @@ def pipeline_run_task(
db.commit()
return
+ run.status = "running"
+ run.started_at = datetime.now(timezone.utc)
+ _set_pipeline_status_if_latest(db, pl, run, "running")
+ db.commit()
+
# ── DAG 编译 ──────────────────────────────────────────────
definition = pl.definition
plan = compile_definition(definition)
@@ -433,30 +616,40 @@ def set_node_status(nid: str, status: str):
else:
input_dataset_version_ids = list(input_dataset_version_ids)
lineage_version_ids = list(input_dataset_version_ids)
+ pinned_versions = _resolve_pinned_input_versions(
+ db, sources, input_dataset_version_ids,
+ )
if input_dataset_version_ids:
run.dataset_version_id = input_dataset_version_ids[0]
db.commit()
transform_route, runtime_spec = _pipeline_runtime_config(pl)
- if sources and not pl.source_dataset_id:
- pl.source_dataset_id = sources[0]["dataset_id"]
- db.commit()
-
outputs = []
multi_source = len(sources) > 1
for source in sources:
+ pinned_version = pinned_versions.get(id(source))
+ if source.get("connection_id"):
+ source_version = _snapshot_database_source(
+ db, svc, source, input_dataset_version_ids,
+ selected_version=pinned_version,
+ )
+ else:
+ source_version = pinned_version
+ # SQL source identity is only known after its first snapshot.
+ # A multi-source pipeline cannot be represented by this one-source
+ # refresh binding; never silently bind it to the first input.
+ if not multi_source and not pl.source_dataset_id and source.get("dataset_id"):
+ pl.source_dataset_id = source["dataset_id"]
+ db.commit()
source["route"] = _source_runtime_route(source, transform_route, pl.route)
- source_version_ids = []
- for version_id in input_dataset_version_ids:
- version = db.get(DatasetVersion, version_id)
- if version is not None and version.dataset_id == source["dataset_id"]:
- source_version_ids.append(version.id)
- source_version = select_pinned_dataset_version(
- db,
- dataset_id=source["dataset_id"],
- input_dataset_version_ids=source_version_ids or None,
- )
+ source_version_ids = [source_version.id] if pinned_version is not None else []
+ if source_version is None:
+ source_version = select_pinned_dataset_version(
+ db,
+ dataset_id=source["dataset_id"],
+ input_dataset_version_ids=source_version_ids or None,
+ )
data = _load_source_rows(
db, svc, source,
input_dataset_version_ids=source_version_ids or None,
@@ -544,17 +737,20 @@ def set_node_status(nid: str, status: str):
"curated_dataset_id": curated_ids[0] if curated_ids else None,
"curated_dataset_ids": curated_ids,
}
+ _set_pipeline_status_if_latest(db, pl, run, "published")
db.commit()
except Exception as e:
- logger.error(f"Pipeline run failed: {e}")
+ detail = safe_connection_error(e)
+ logger.error("Pipeline run failed: %s", detail)
if run:
run.status = "failed"
- run.error_log = str(e)
+ run.error_log = detail
run.finished_at = datetime.now(timezone.utc)
stats = dict(run.stats or {})
stats.setdefault("node_status", {})
run.stats = stats
+ _set_pipeline_status_if_latest(db, pl, run, "failed")
db.commit()
finally:
db.close()
diff --git a/backend/app/tasks/v2/refresh_tasks.py b/backend/app/tasks/v2/refresh_tasks.py
index b5b2138a..6bd262e6 100644
--- a/backend/app/tasks/v2/refresh_tasks.py
+++ b/backend/app/tasks/v2/refresh_tasks.py
@@ -27,12 +27,6 @@
logger = logging.getLogger(__name__)
-# Placeholder resource scope for a connection/pipeline-level manual or
-# scheduled refresh until multi-resource fan-out lands (matches
-# schedule_service.DEFAULT_RESOURCE).
-DEFAULT_RESOURCE = "__scheduled__"
-
-
def _celery_send_task(task_name: str, args, queue: str) -> str:
"""Adapter matching `enqueue_refresh_run`'s `send_task` contract —
Celery's own `send_task(name, args, kwargs, ...)` would otherwise bind a
@@ -325,27 +319,28 @@ def create_manual_connection_run(db, connection_id: str):
"""
from app.models.v2.connection import Connection
from app.schemas.refresh import RefreshError, RefreshPolicy
- from app.services.v2.incremental.operations import _policy_for_source, _source_state
+ from app.services.v2.incremental.operations import (
+ _policy_for_source,
+ _source_state,
+ resolve_refresh_resource,
+ )
from app.services.v2.incremental.contract import claim_refresh_run
connection = db.get(Connection, connection_id)
if connection is None:
- # Preserve the historical internal helper behavior for callers that
- # already provision source state without a Connection row. The
- # authenticated route verifies the connection first; persisted
- # policies are enforced whenever that source record exists.
- policy = RefreshPolicy.BATCH
- else:
- configured_state = _source_state(db, source_id=connection_id)
- policy = _policy_for_source(connection, configured_state)
- if policy is RefreshPolicy.EVENT_DRIVEN:
- raise RefreshError(
- "EVENT_TRIGGER_REQUIRES_SIGNATURE",
- "event-driven refreshes require the signed webhook route",
- )
+ raise RefreshError("SOURCE_NOT_FOUND", f"no source connection {connection_id}")
+
+ resource = resolve_refresh_resource(db, source_id=connection_id)
+ configured_state = _source_state(db, source_id=connection_id, resource=resource)
+ policy = _policy_for_source(connection, configured_state)
+ if policy is RefreshPolicy.EVENT_DRIVEN:
+ raise RefreshError(
+ "EVENT_TRIGGER_REQUIRES_SIGNATURE",
+ "event-driven refreshes require the signed webhook route",
+ )
run = claim_refresh_run(
- db, source_id=connection_id, resource=DEFAULT_RESOURCE, policy=policy,
+ db, source_id=connection_id, resource=resource, policy=policy,
idempotency_key=f"manual:{uuid.uuid4()}", lease_owner="manual-sync",
now=datetime.now(timezone.utc),
)
diff --git "a/backend/evals/document_extraction/ground_truth/\344\276\233\345\272\224\351\223\276.entities.json" "b/backend/evals/document_extraction/ground_truth/\344\276\233\345\272\224\351\223\276.entities.json"
new file mode 100644
index 00000000..c992b477
--- /dev/null
+++ "b/backend/evals/document_extraction/ground_truth/\344\276\233\345\272\224\351\223\276.entities.json"
@@ -0,0 +1,17 @@
+[
+ "供应商",
+ "供应商分级",
+ "库存",
+ "物料",
+ "安全库存",
+ "采购申请",
+ "采购审批",
+ "质量控制",
+ "来料检验",
+ "供应商审计",
+ "物流",
+ "配送",
+ "承运商",
+ "质检部",
+ "董事会"
+]
diff --git "a/backend/evals/document_extraction/ground_truth/\344\277\241\350\264\267.entities.json" "b/backend/evals/document_extraction/ground_truth/\344\277\241\350\264\267.entities.json"
new file mode 100644
index 00000000..d0709fdb
--- /dev/null
+++ "b/backend/evals/document_extraction/ground_truth/\344\277\241\350\264\267.entities.json"
@@ -0,0 +1,24 @@
+[
+ "借款人",
+ "金融机构资金方",
+ "信贷科技平台",
+ "信贷驱动服务",
+ "平台服务",
+ "重资本模式",
+ "轻资本模式",
+ "信用分层",
+ "信用评分",
+ "授信额度",
+ "授信审批",
+ "逾期阶段",
+ "贷后催收",
+ "风险准备金",
+ "获客渠道",
+ "转化漏斗",
+ "风险定价模型",
+ "客户生命周期",
+ "合规部",
+ "风控委员会",
+ "反欺诈黑名单",
+ "多头借贷指标"
+]
diff --git "a/backend/evals/document_extraction/ground_truth/\345\214\273\347\226\227.entities.json" "b/backend/evals/document_extraction/ground_truth/\345\214\273\347\226\227.entities.json"
new file mode 100644
index 00000000..cb492fcb
--- /dev/null
+++ "b/backend/evals/document_extraction/ground_truth/\345\214\273\347\226\227.entities.json"
@@ -0,0 +1,19 @@
+[
+ "高血压",
+ "2型糖尿病",
+ "收缩压",
+ "舒张压",
+ "空腹血糖",
+ "糖化血红蛋白",
+ "肾功能",
+ "ACEI",
+ "ARB",
+ "CCB",
+ "二甲双胍",
+ "胰岛素",
+ "糖尿病肾病",
+ "视网膜病变",
+ "低血糖",
+ "药物相互作用",
+ "用药重整"
+]
diff --git "a/backend/evals/document_extraction/ground_truth/\350\264\242\345\212\241.entities.json" "b/backend/evals/document_extraction/ground_truth/\350\264\242\345\212\241.entities.json"
new file mode 100644
index 00000000..2827b0ca
--- /dev/null
+++ "b/backend/evals/document_extraction/ground_truth/\350\264\242\345\212\241.entities.json"
@@ -0,0 +1,17 @@
+[
+ "供应商",
+ "发票",
+ "采购订单",
+ "采购审批",
+ "付款",
+ "费用报销",
+ "预算",
+ "预算调整",
+ "应收账款",
+ "账龄",
+ "坏账准备",
+ "固定资产",
+ "资产折旧",
+ "审计委员会",
+ "董事会"
+]
diff --git a/backend/evals/document_extraction/manifest/4domains.json b/backend/evals/document_extraction/manifest/4domains.json
new file mode 100644
index 00000000..a6353945
--- /dev/null
+++ b/backend/evals/document_extraction/manifest/4domains.json
@@ -0,0 +1,6 @@
+[
+ {"case_id": "credit-01", "domain": "信贷", "source_file": "test_data/信贷/信贷业务战略.md", "ground_truth_id": "信贷"},
+ {"case_id": "supply-chain-01", "domain": "供应链", "source_file": "test_data/供应链/supply_chain_strategy.md", "ground_truth_id": "供应链"},
+ {"case_id": "medical-01", "domain": "医疗", "source_file": "test_data/医疗/clinical_protocols.md", "ground_truth_id": "医疗"},
+ {"case_id": "finance-01", "domain": "财务", "source_file": "test_data/财务/financial_controls.md", "ground_truth_id": "财务"}
+]
diff --git a/backend/evals/document_extraction/run.py b/backend/evals/document_extraction/run.py
index 2a072e3a..6fd5f73b 100644
--- a/backend/evals/document_extraction/run.py
+++ b/backend/evals/document_extraction/run.py
@@ -4,7 +4,9 @@
import json
import os
-from evals.document_extraction.validators import dedup_gate, instance_leakage_gate, keyword_recall
+from evals.document_extraction.validators import (
+ dedup_gate, instance_leakage_gate, keyword_recall, precision_recall, rule_leakage_gate,
+)
def run_eval(manifest_path: str, ground_truth_dir: str, adapter, repo_root: str) -> dict:
@@ -17,6 +19,15 @@ def run_eval(manifest_path: str, ground_truth_dir: str, adapter, repo_root: str)
with open(gt_path, encoding="utf-8") as f:
ground_truth = json.load(f)
+ # optional: a full gold entity list for real precision/recall/F1,
+ # separate from the keyword-hint ground_truth above — a domain
+ # without this file simply skips precision/recall, no breakage
+ entities_gt_path = os.path.join(ground_truth_dir, f"{case['ground_truth_id']}.entities.json")
+ gold_entities = None
+ if os.path.exists(entities_gt_path):
+ with open(entities_gt_path, encoding="utf-8") as f:
+ gold_entities = json.load(f)
+
source_path = os.path.join(repo_root, case["source_file"])
try:
extraction_result = adapter.extract(source_path)
@@ -26,7 +37,10 @@ def run_eval(manifest_path: str, ground_truth_dir: str, adapter, repo_root: str)
"keyword_recall": keyword_recall(ground_truth, extraction_result),
"dedup_findings": dedup_gate(extraction_result),
"leakage_findings": instance_leakage_gate(extraction_result),
+ "rule_leakage_findings": rule_leakage_gate(extraction_result),
}
+ if gold_entities is not None:
+ case_entry["precision_recall"] = precision_recall(gold_entities, extraction_result)
except Exception as e:
case_entry = {
"case_id": case["case_id"],
@@ -41,11 +55,16 @@ def run_eval(manifest_path: str, ground_truth_dir: str, adapter, repo_root: str)
sum(c["keyword_recall"]["score"] for c in recall_cases) / len(recall_cases)
if recall_cases else 0.0
)
+ pr_cases = [c for c in case_results if "precision_recall" in c]
+ mean_f1 = sum(c["precision_recall"]["f1"] for c in pr_cases) / len(pr_cases) if pr_cases else None
summary = {
"total_cases": total,
"mean_keyword_recall": mean_recall,
"cases_with_dedup_findings": sum(1 for c in case_results if c.get("dedup_findings")),
"cases_with_leakage_findings": sum(1 for c in case_results if c.get("leakage_findings")),
+ "cases_with_rule_leakage_findings": sum(1 for c in case_results if c.get("rule_leakage_findings")),
+ "cases_with_precision_recall": len(pr_cases),
+ "mean_f1": mean_f1,
}
return {"cases": case_results, "summary": summary}
diff --git a/backend/evals/document_extraction/validators.py b/backend/evals/document_extraction/validators.py
index c3b311fe..2e545313 100644
--- a/backend/evals/document_extraction/validators.py
+++ b/backend/evals/document_extraction/validators.py
@@ -13,6 +13,17 @@
_ID_PATTERN = re.compile(r"^[A-Za-z]+\d+$")
_TRAILING_DIGITS_PATTERN = re.compile(r"\d{3,}$")
+# A letter (+ optional 1-2 digits) followed by a small closed set of
+# tier/stage/round markers, and nothing else — the shape of an enumeration
+# VALUE of some concept's property (信用分层's "A层", 逾期阶段's "M2+",
+# 融资轮次's "D轮融资"), not a concept in its own right. Kept in sync with
+# app.services.llm_service._looks_like_enum_value.
+_ENUM_VALUE_PATTERN = re.compile(
+ r"^[A-Za-z]{1,2}[+-]?(轮融资|阶段|层|级|档|类)$|^M\d{1,2}\+?(阶段)?$"
+)
+
+_RULE_NAME_SUFFIXES = ("规则", "政策", "办法", "制度", "准则", "细则", "规程")
+
def _all_text_blobs(extraction_result: dict) -> list[str]:
"""Every name_cn/description/entity_type string across all categories,
@@ -44,6 +55,51 @@ def keyword_recall(ground_truth: list[dict], extraction_result: dict) -> dict:
return {"score": score, "passed": passed, "failed": failed}
+def precision_recall(gold_entities: list[str], extraction_result: dict, fuzzy_threshold: int = 85) -> dict:
+ """Real precision/recall/F1 against a full gold-labeled entity list (not
+ keyword_recall's loose "does this keyword appear somewhere" check).
+ Fuzzy-matched (rapidfuzz, same threshold convention as dedup_gate) so a
+ minor surface-form difference between the gold name and the extracted
+ name_cn doesn't count as a miss — each gold entity and each extracted
+ entity is consumed by at most one match, so duplicates on either side
+ can't inflate the score."""
+ extracted_names = [
+ str(item.get("name_cn") or "") for item in (extraction_result.get("entities") or [])
+ if isinstance(item, dict) and item.get("name_cn")
+ ]
+ unmatched_extracted = list(extracted_names)
+ matched_gold: list[str] = []
+ missed_gold: list[str] = []
+ for gold_name in gold_entities:
+ best_idx, best_score = None, -1
+ for idx, candidate in enumerate(unmatched_extracted):
+ score = fuzz.ratio(gold_name, candidate)
+ if score > best_score:
+ best_idx, best_score = idx, score
+ if best_idx is not None and best_score >= fuzzy_threshold:
+ matched_gold.append(gold_name)
+ unmatched_extracted.pop(best_idx)
+ else:
+ missed_gold.append(gold_name)
+
+ true_positive = len(matched_gold)
+ false_negative = len(missed_gold)
+ false_positive = len(unmatched_extracted) # extracted but never matched a gold entity
+ precision = true_positive / (true_positive + false_positive) if (true_positive + false_positive) else 1.0
+ recall = true_positive / (true_positive + false_negative) if (true_positive + false_negative) else 1.0
+ f1 = (2 * precision * recall / (precision + recall)) if (precision + recall) else 0.0
+ return {
+ "precision": round(precision, 3),
+ "recall": round(recall, 3),
+ "f1": round(f1, 3),
+ "true_positive": true_positive,
+ "false_positive": false_positive,
+ "false_negative": false_negative,
+ "missed_gold_entities": missed_gold,
+ "unmatched_extracted_entities": unmatched_extracted,
+ }
+
+
def dedup_gate(extraction_result: dict, threshold: int = 85) -> list[dict]:
findings: list[dict] = []
for category in ("entities", "logic_rules", "actions"):
@@ -67,12 +123,33 @@ def _looks_like_instance_id(name: str) -> bool:
return bool(_ID_PATTERN.match(name)) or bool(_TRAILING_DIGITS_PATTERN.search(name))
+def _looks_like_enum_value(name: str) -> bool:
+ return bool(_ENUM_VALUE_PATTERN.match(name))
+
+
def instance_leakage_gate(extraction_result: dict) -> list[dict]:
findings: list[dict] = []
for item in extraction_result.get("entities") or []:
if not isinstance(item, dict):
continue
name = str(item.get("name_cn") or "")
- if name and _looks_like_instance_id(name):
+ if not name:
+ continue
+ if _looks_like_instance_id(name):
findings.append({"name": name, "reason": "matches ID/numbered-record pattern"})
+ elif _looks_like_enum_value(name):
+ findings.append({"name": name, "reason": "matches tier/stage/round enum-value pattern"})
+ return findings
+
+
+def rule_leakage_gate(extraction_result: dict) -> list[dict]:
+ """A business rule/policy name (e.g. "促销定价规则") is not an entity —
+ it belongs in logic_rules."""
+ findings: list[dict] = []
+ for item in extraction_result.get("entities") or []:
+ if not isinstance(item, dict):
+ continue
+ name = str(item.get("name_cn") or "")
+ if name and name.endswith(_RULE_NAME_SUFFIXES):
+ findings.append({"name": name, "reason": "matches business-rule/policy name pattern"})
return findings
diff --git a/backend/openapi-agent.json b/backend/openapi-agent.json
index c43d7de1..f0bfbdb2 100644
--- a/backend/openapi-agent.json
+++ b/backend/openapi-agent.json
@@ -14617,6 +14617,119 @@
]
}
},
+ "/api/v2/connections/{connection_id}/resources": {
+ "get": {
+ "description": "List tables/collections exposed by a saved connection.",
+ "operationId": "list_connection_resources_api_v2_connections__connection_id__resources_get",
+ "parameters": [
+ {
+ "in": "path",
+ "name": "connection_id",
+ "required": true,
+ "schema": {
+ "title": "Connection Id",
+ "type": "string"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "content": {
+ "application/json": {
+ "schema": {}
+ }
+ },
+ "description": "Successful Response"
+ },
+ "422": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ },
+ "description": "Validation Error"
+ }
+ },
+ "security": [
+ {
+ "HTTPBearer": []
+ }
+ ],
+ "summary": "List Connection Resources",
+ "tags": [
+ "v2-connections"
+ ]
+ }
+ },
+ "/api/v2/connections/{connection_id}/resources/{resource}/sample": {
+ "get": {
+ "description": "Read a bounded sample from one saved connection resource.",
+ "operationId": "sample_connection_resource_api_v2_connections__connection_id__resources__resource__sample_get",
+ "parameters": [
+ {
+ "in": "path",
+ "name": "connection_id",
+ "required": true,
+ "schema": {
+ "title": "Connection Id",
+ "type": "string"
+ }
+ },
+ {
+ "in": "path",
+ "name": "resource",
+ "required": true,
+ "schema": {
+ "title": "Resource",
+ "type": "string"
+ }
+ },
+ {
+ "in": "query",
+ "name": "limit",
+ "required": false,
+ "schema": {
+ "default": 20,
+ "maximum": 1000,
+ "minimum": 1,
+ "title": "Limit",
+ "type": "integer"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "content": {
+ "application/json": {
+ "schema": {}
+ }
+ },
+ "description": "Successful Response"
+ },
+ "422": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ },
+ "description": "Validation Error"
+ }
+ },
+ "security": [
+ {
+ "HTTPBearer": []
+ }
+ ],
+ "summary": "Sample Connection Resource",
+ "tags": [
+ "v2-connections"
+ ]
+ }
+ },
"/api/v2/connections/{connection_id}/schedule": {
"post": {
"description": "Compatibility delegate for the typed refresh schedule operation.",
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index c2826a3f..e41e1c44 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -2,7 +2,7 @@
name = "ontexus-backend"
version = "0.1.0"
description = "Ontexus backend service"
-requires-python = ">=3.11"
+requires-python = ">=3.11,<3.13"
[tool.setuptools.packages.find]
include = ["app*"]
diff --git a/backend/scripts/check_python_version.py b/backend/scripts/check_python_version.py
index eeec27e6..cf0fc6db 100644
--- a/backend/scripts/check_python_version.py
+++ b/backend/scripts/check_python_version.py
@@ -3,15 +3,18 @@
import sys
MIN_SUPPORTED = (3, 11)
+MAX_SUPPORTED_EXCLUSIVE = (3, 13)
UNSUPPORTED_PYTHON_VERSION = "UNSUPPORTED_PYTHON_VERSION"
def require_supported_python():
- if sys.version_info[:2] < MIN_SUPPORTED:
+ if sys.version_info[:2] < MIN_SUPPORTED or sys.version_info[:2] >= MAX_SUPPORTED_EXCLUSIVE:
detected = ".".join(str(part) for part in sys.version_info[:3])
- required = ".".join(str(part) for part in MIN_SUPPORTED)
+ minimum = ".".join(str(part) for part in MIN_SUPPORTED)
+ maximum = ".".join(str(part) for part in MAX_SUPPORTED_EXCLUSIVE)
raise SystemExit(
- f"{UNSUPPORTED_PYTHON_VERSION}: detected Python {detected}, required >= {required}"
+ f"{UNSUPPORTED_PYTHON_VERSION}: detected Python {detected}, "
+ f"required >= {minimum}, < {maximum}"
)
return True
diff --git a/backend/tests/agent/test_core_route_registration.py b/backend/tests/agent/test_core_route_registration.py
index ddbece72..f7f5f8eb 100644
--- a/backend/tests/agent/test_core_route_registration.py
+++ b/backend/tests/agent/test_core_route_registration.py
@@ -475,6 +475,8 @@ def test_idempotency_migration_upgrade_downgrade_cycle():
def test_audit_route_rejects_unknown_model_id_422(idem_schema):
from app.routers.audit import router as audit_router
from app.routers.audit import start_audit # noqa: F401 — route-local fix surface
+ from app.models.user import User
+ from app.services.ontology_access import creator_grant
session_factory = sessionmaker(bind=create_engine(_scoped_url(idem_schema)))
session = session_factory()
@@ -491,6 +493,7 @@ def test_audit_route_rejects_unknown_model_id_422(idem_schema):
"VALUES (:id, 'p', 'test', 'v0.1', 'created', :by, now(), now(), :domain, 1)"
), {"id": "onto-1", "by": admin_id, "domain": DEFAULT_DOMAIN})
session.commit()
+ creator_grant(session, "onto-1", session.get(User, admin_id))
finally:
session.close()
diff --git a/backend/tests/agent/test_extraction_normalization.py b/backend/tests/agent/test_extraction_normalization.py
index 43ba1907..0edae9a7 100644
--- a/backend/tests/agent/test_extraction_normalization.py
+++ b/backend/tests/agent/test_extraction_normalization.py
@@ -15,10 +15,13 @@
from app.services.llm_service import (
VAGUE_RELATION_TYPES,
+ _looks_like_enum_value,
+ _looks_like_rule_name,
apply_entity_resolution,
extract_ontology,
normalize_extracted_ontology,
resolve_entities,
+ summarize_entity,
)
BACKEND_DIR = Path(__file__).resolve().parents[2]
@@ -39,7 +42,10 @@ def test_extraction_discipline_red_contract():
def test_normalization_dedups_and_drops_junk_over_test_data_document():
"""A noisy extraction over the real 信贷 strategy document is normalized:
duplicate surface forms merge, junk entities vanish, dangling and vague
- relations are dropped, hierarchy relations survive."""
+ relations are dropped, hierarchy relations survive. Enum-value labels
+ (tier/stage/round markers) and business-rule names are also entity
+ leakage — dropped, along with any relation left dangling by their
+ removal."""
document = TEST_DATA / "信贷" / "信贷业务战略.md"
if not document.exists():
pytest.skip("test_data document unavailable")
@@ -54,6 +60,7 @@ def test_normalization_dedups_and_drops_junk_over_test_data_document():
{"name_cn": "A层", "type": "Category", "description": "优质客户", "properties": {}},
{"name_cn": "B层", "type": "Category", "description": "良好客户", "properties": {}},
{"name_cn": "授信额度", "type": "Concept", "description": "可贷上限", "properties": {}},
+ {"name_cn": "促销定价规则", "type": "Concept", "description": "误提取的规则", "properties": {}},
{"name_cn": "12345", "type": "Concept", "description": "噪音", "properties": {}},
{"name_cn": "", "type": "Concept", "description": "空名", "properties": {}},
{"name_cn": "信贷业务战略.md", "type": "Concept", "description": "文件名", "properties": {}},
@@ -85,6 +92,11 @@ def test_normalization_dedups_and_drops_junk_over_test_data_document():
# junk entities dropped
assert "12345" not in names
assert "信贷业务战略.md" not in names
+ # enum-value labels (tier markers) are not concepts — dropped
+ assert "A层" not in names
+ assert "B层" not in names
+ # a business rule is not an entity — dropped (belongs in logic_rules)
+ assert "促销定价规则" not in names
assert not any(not (e.get("name_cn") or "").strip() for e in normalized["entities"])
# every surviving entity carries a grounded one-line description
assert all(e.get("description") for e in normalized["entities"])
@@ -97,10 +109,31 @@ def test_normalization_dedups_and_drops_junk_over_test_data_document():
assert ("借款人", "DEPENDS_ON", "信用评分") in triples
# the duplicate DEPENDS_ON appeared twice -> deduplicated to one
assert sum(1 for r in normalized["relations"] if r["type"] == "DEPENDS_ON") == 1
- # hierarchy relations (IS-A / INSTANCE-OF) survive intact
- assert ("A层", "INSTANCE-OF", "信用分层") in triples
- assert ("B层", "INSTANCE-OF", "信用分层") in triples
- assert ("A层", "GOVERNED_BY", "授信额度") in triples
+ # relations pointing at the dropped enum-value entities are now dangling
+ # references and are dropped too
+ assert ("A层", "INSTANCE-OF", "信用分层") not in triples
+ assert ("B层", "INSTANCE-OF", "信用分层") not in triples
+ assert ("A层", "GOVERNED_BY", "授信额度") not in triples
+
+
+def test_looks_like_enum_value_matches_tier_stage_and_round_labels():
+ for name in ("A层", "B层", "C级", "M0", "M1", "M2+", "M0阶段", "A轮融资", "D轮融资", "A档", "B类"):
+ assert _looks_like_enum_value(name), name
+
+
+def test_looks_like_enum_value_does_not_match_real_concepts():
+ for name in ("借款人", "供应商", "A股", "P7", "信用分层", "授信额度", "D轮"):
+ assert not _looks_like_enum_value(name), name
+
+
+def test_looks_like_rule_name_matches_policy_and_rule_suffixes():
+ for name in ("促销定价规则", "审批超时规则", "员工考勤制度", "风险准入政策"):
+ assert _looks_like_rule_name(name), name
+
+
+def test_looks_like_rule_name_does_not_match_concepts():
+ for name in ("借款人", "信用分层", "贷款产品"):
+ assert not _looks_like_rule_name(name), name
def test_normalization_is_idempotent_and_keeps_clean_output():
@@ -208,6 +241,67 @@ def test_resolve_entities_single_type_member_skips_llm_call():
assert alias_map == {"唯一实体": "唯一实体"}
+def test_resolve_entities_anchors_canonical_to_existing_entity():
+ """Incremental-update guidance: a new-run surface form that clusters with
+ an already-published entity must remap to that entity's own name, not to
+ whatever the model picked as canonical."""
+ from app.services import llm_service
+ original = llm_service._call_llm
+
+ def fake_call(provider, api_key, api_base, model, messages):
+ assert "[已存在]" in messages[1]["content"]
+ return '{"clusters": [{"canonical": "客户方", "aliases": ["客户方", "客户"]}]}'
+
+ llm_service._call_llm = fake_call
+ try:
+ new_entities = [{"name_cn": "客户方", "type": "Concept", "description": "本轮新提取"}]
+ existing = [{"name_cn": "客户", "type": "Concept", "description": "已发布的规范实体"}]
+ alias_map = resolve_entities(
+ new_entities, {"provider": "openai", "api_key": "x"}, "m", existing_entities=existing,
+ )
+ finally:
+ llm_service._call_llm = original
+ # the model picked "客户方" as canonical, but the existing entity "客户"
+ # must win regardless
+ assert alias_map["客户方"] == "客户"
+
+
+def test_resolve_entities_single_new_entity_still_checked_against_existing():
+ """A single new entity (which would otherwise skip clustering entirely)
+ must still be checked against existing entities of the same type."""
+ from app.services import llm_service
+ original = llm_service._call_llm
+ llm_service._call_llm = lambda *a, **k: (
+ '{"clusters": [{"canonical": "借款人", "aliases": ["借款方", "借款人"]}]}'
+ )
+ try:
+ alias_map = resolve_entities(
+ [{"name_cn": "借款方", "type": "Concept", "description": "本轮新提取"}],
+ {"provider": "openai", "api_key": "x"}, "m",
+ existing_entities=[{"name_cn": "借款人", "type": "Concept", "description": "已发布"}],
+ )
+ finally:
+ llm_service._call_llm = original
+ assert alias_map["借款方"] == "借款人"
+
+
+def test_resolve_entities_does_not_touch_existing_entities_of_other_types():
+ """Existing entities of a different type never enter the same clustering
+ call — no cross-type contamination."""
+ from app.services import llm_service
+ original = llm_service._call_llm
+ llm_service._call_llm = lambda *a, **k: (_ for _ in ()).throw(AssertionError("should not be called"))
+ try:
+ alias_map = resolve_entities(
+ [{"name_cn": "唯一实体", "type": "Concept", "description": "d"}],
+ {"provider": "openai", "api_key": "x"}, "m",
+ existing_entities=[{"name_cn": "供应商甲", "type": "Supplier", "description": "d"}],
+ )
+ finally:
+ llm_service._call_llm = original
+ assert alias_map == {"唯一实体": "唯一实体"}
+
+
def test_apply_entity_resolution_merges_and_remaps_references():
"""apply_entity_resolution merges aliased entities (keeping the richer
description), remaps relation endpoints, drops self-loops created by the
@@ -255,3 +349,46 @@ def test_apply_entity_resolution_is_noop_for_identity_map():
result = {"entities": [{"name_cn": "A"}], "relations": []}
assert apply_entity_resolution(result, {"A": "A"}) is result
assert apply_entity_resolution(result, {}) is result
+
+
+def test_summarize_entity_parses_profile_from_llm_response():
+ from app.services import llm_service
+ original = llm_service._call_llm
+ llm_service._call_llm = lambda *a, **k: (
+ '{"summary": "借款人是信贷业务的核心概念。", '
+ '"key_facts": ["借款人需通过授信审批", "借款人拥有贷款账户", "多出来的第四条", '
+ '"第五条", "第六条会被截断"], '
+ '"time_range": {"start": "2026", "end": "ongoing"}}'
+ )
+ try:
+ profile = summarize_entity(
+ "借款人", "贷款申请人", "(借款人) --[APPLIES_FOR]--> (贷款)",
+ {"provider": "openai", "api_key": "x"}, "m",
+ )
+ finally:
+ llm_service._call_llm = original
+ assert profile["summary"] == "借款人是信贷业务的核心概念。"
+ assert len(profile["key_facts"]) == 5 # capped at 5
+ assert profile["time_range"] == {"start": "2026", "end": "ongoing"}
+
+
+def test_summarize_entity_returns_none_on_failure():
+ from app.services import llm_service
+ original = llm_service._call_llm
+ llm_service._call_llm = lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom"))
+ try:
+ profile = summarize_entity("借款人", "d", "", {"provider": "openai", "api_key": "x"}, "m")
+ finally:
+ llm_service._call_llm = original
+ assert profile is None
+
+
+def test_summarize_entity_defaults_missing_time_range_to_unknown():
+ from app.services import llm_service
+ original = llm_service._call_llm
+ llm_service._call_llm = lambda *a, **k: '{"summary": "s", "key_facts": []}'
+ try:
+ profile = summarize_entity("X", "d", "", {"provider": "openai", "api_key": "x"}, "m")
+ finally:
+ llm_service._call_llm = original
+ assert profile["time_range"] == {"start": "unknown", "end": "unknown"}
diff --git a/backend/tests/agent/test_import_mutation_closure.py b/backend/tests/agent/test_import_mutation_closure.py
index a83d8e6d..42574636 100644
--- a/backend/tests/agent/test_import_mutation_closure.py
+++ b/backend/tests/agent/test_import_mutation_closure.py
@@ -133,8 +133,13 @@ def test_zz_extraction_rejects_unknown_model_with_stable_error(imports_db):
from app.routers import extraction
from app.services.auth_service import create_access_token
+ from app.services.ontology_access import backfill_creator_grants
+
Session, _ = imports_db
actor_id, ontology_id = _seed(imports_db)
+ # _seed inserts via direct SQL, bypassing create_ontology's creator grant
+ with Session() as session:
+ backfill_creator_grants(session)
token = create_access_token({"sub": actor_id, "role": "editor"})
headers = {"Authorization": f"Bearer {token}"}
diff --git a/backend/tests/agent/test_legacy_route_closure.py b/backend/tests/agent/test_legacy_route_closure.py
index be709588..b7bd9294 100644
--- a/backend/tests/agent/test_legacy_route_closure.py
+++ b/backend/tests/agent/test_legacy_route_closure.py
@@ -151,6 +151,8 @@ def closure_db():
def _seed(closure_db):
from app.services.auth_service import hash_password
+ from app.models.user import User
+ from app.services.ontology_access import creator_grant
Session, _ = closure_db
username = "closure-" + uuid.uuid4().hex[:8]
@@ -167,6 +169,7 @@ def _seed(closure_db):
), {"id": str(uuid.uuid4()), "name": "Closure " + uuid.uuid4().hex[:8], "creator": actor_id, "domain": DEFAULT_DOMAIN})
ontology_id = session.execute(text("SELECT id FROM ontology_projects WHERE created_by=:c ORDER BY created_at DESC LIMIT 1"), {"c": actor_id}).scalar_one()
session.commit()
+ creator_grant(session, ontology_id, session.get(User, actor_id))
return actor_id, ontology_id
diff --git a/backend/tests/agent/test_legacy_runtime_tables.py b/backend/tests/agent/test_legacy_runtime_tables.py
index ae82eae0..577fc088 100644
--- a/backend/tests/agent/test_legacy_runtime_tables.py
+++ b/backend/tests/agent/test_legacy_runtime_tables.py
@@ -160,8 +160,11 @@ def test_audit_and_instance_writes_no_longer_500(legacy_schema):
from app.deps import get_db
from app.routers import audit as audit_router
from app.services.auth_service import create_access_token
+ from app.services.ontology_access import backfill_creator_grants
with Session() as session:
+ # the ontology was inserted via direct SQL, bypassing create_ontology's creator grant
+ backfill_creator_grants(session)
actor_id = session.execute(text("SELECT id FROM users WHERE username='lr-user'")).scalar_one()
token = create_access_token({"sub": actor_id, "role": "admin"})
diff --git a/backend/tests/agent/test_model_caller_cutover.py b/backend/tests/agent/test_model_caller_cutover.py
index 07f075aa..3aedbc2f 100644
--- a/backend/tests/agent/test_model_caller_cutover.py
+++ b/backend/tests/agent/test_model_caller_cutover.py
@@ -178,6 +178,57 @@ def _headers(token):
return {"Authorization": f"Bearer {token}"}
+@pytest.mark.parametrize("reference", [None, "agent", "prompt"])
+def test_delete_model_preserves_migrated_reference_guards(pg, reference):
+ from fastapi import HTTPException
+ from app.models.agent import Agent, AgentVersion
+ from app.models.agent_config import PromptGeneration
+ from app.models.model_version import ModelMigrationFinding
+ from app.routers.models import delete_model
+
+ model_id = _seed_llm_config(pg)
+ _migrate_rows(pg)
+ version_id = pg.execute(text(
+ "SELECT active_version_id FROM model_configs WHERE id = :id"
+ ), {"id": model_id}).scalar_one()
+ assert version_id is not None
+ pg.add(ModelMigrationFinding(model_config_id=model_id, code="TEST", reason="test finding"))
+ if reference:
+ pg.add(Agent(id="delete-test-agent", owner_id="seed-user"))
+ pg.flush()
+ if reference == "agent":
+ schema_id = pg.execute(text(
+ "SELECT active_version_id FROM application_state_schema_registries WHERE application_key = 'chat-v1'"
+ )).scalar_one()
+ pg.add(AgentVersion(
+ agent_id="delete-test-agent", version_no=1, name="dependent",
+ default_model_config_version_id=version_id, default_model_name="gpt-4o",
+ application_state_schema_version_id=schema_id, config_hash="a" * 64,
+ created_by="seed-user",
+ ))
+ else:
+ pg.add(PromptGeneration(
+ agent_id="delete-test-agent", base_version_no=1,
+ model_config_version_id=version_id, model_name="gpt-4o",
+ input_hash="b" * 64, output_hash="c" * 64, requester_id="seed-user",
+ ))
+ pg.commit()
+
+ for table in ("model_config_versions", "model_credentials", "model_migration_findings"):
+ assert pg.execute(text(f"SELECT count(*) FROM {table} WHERE model_config_id = :id"), {"id": model_id}).scalar_one() > 0
+ if reference:
+ with pytest.raises(HTTPException) as denied:
+ delete_model(model_id, db=pg)
+ assert denied.value.status_code == 409
+ assert denied.value.detail == "MODEL_REFERENCED"
+ assert pg.execute(text("SELECT active_version_id FROM model_configs WHERE id = :id"), {"id": model_id}).scalar_one() == version_id
+ else:
+ delete_model(model_id, db=pg)
+ assert pg.execute(text("SELECT count(*) FROM model_configs WHERE id = :id"), {"id": model_id}).scalar_one() == 0
+ for table in ("model_config_versions", "model_credentials", "model_migration_findings"):
+ assert pg.execute(text(f"SELECT count(*) FROM {table} WHERE model_config_id = :id"), {"id": model_id}).scalar_one() == 0
+
+
def test_port_resolves_immutable_llm_version_and_rejects_blocked(pg):
from app.services.publication.extraction_model_port import SqlExtractionModelPort
@@ -277,8 +328,10 @@ def test_model_api_redacted_tagged_union_and_versioned_surface(pg):
assert detail["active_version"]["version_no"] == 2
assert detail["active_version"]["behavior_hash"] != versions[0]["behavior_hash"]
- # delete is RESTRICT while versions exist
- assert client.delete(f"/api/v1/models/{llm_id}", headers=headers).status_code == 409
+ # Owned versions alone do not make an unused model undeletable. Real
+ # Agent/prompt references are covered separately by the deletion matrix.
+ assert client.delete(f"/api/v1/models/{llm_id}", headers=headers).status_code == 204
+ assert client.get(f"/api/v1/models/{llm_id}", headers=headers).status_code == 404
# viewer cannot create/update models (editor ceiling)
assert client.post("/api/v1/models", json={"name": "x", "provider": "openai", "models": ["gpt-4o"]}, headers=viewer_headers).status_code == 403
diff --git a/backend/tests/agent/test_ontology_access_admin.py b/backend/tests/agent/test_ontology_access_admin.py
index 02a65eec..d7f7b077 100644
--- a/backend/tests/agent/test_ontology_access_admin.py
+++ b/backend/tests/agent/test_ontology_access_admin.py
@@ -11,6 +11,7 @@
import hashlib
import json
import os
+from datetime import datetime, timedelta, timezone
from pathlib import Path
import subprocess
import sys
@@ -79,7 +80,7 @@ def access_db():
engine.dispose()
-def _insert_user(Session, username, role="editor", active=True, domain=True):
+def _insert_user(Session, username, role="editor", active=True, domain=True, domain_id=None):
domain_clause = ", security_domain_id" if domain else ""
domain_value = ", :domain" if domain else ""
params = {
@@ -90,7 +91,7 @@ def _insert_user(Session, username, role="editor", active=True, domain=True):
"active": active,
}
if domain:
- params["domain"] = DEFAULT_DOMAIN
+ params["domain"] = domain_id or DEFAULT_DOMAIN
with Session() as session:
session.execute(text(
f"INSERT INTO users (id, username, email, password_hash, role, is_active, created_at, updated_at{domain_clause}) "
@@ -101,12 +102,12 @@ def _insert_user(Session, username, role="editor", active=True, domain=True):
return user_id
-def _insert_ontology(Session, name, created_by):
+def _insert_ontology(Session, name, created_by, domain_id=None):
with Session() as session:
session.execute(text(
"INSERT INTO ontology_projects (id, name, domain, version, status, created_by, created_at, updated_at, security_domain_id) "
"VALUES (:id, :name, 'test', 'v0.1', 'draft', :created_by, now(), now(), :domain)"
- ), {"id": str(uuid.uuid4()), "name": name, "created_by": created_by, "domain": DEFAULT_DOMAIN})
+ ), {"id": str(uuid.uuid4()), "name": name, "created_by": created_by, "domain": domain_id or DEFAULT_DOMAIN})
ontology_id = session.execute(text("SELECT id FROM ontology_projects WHERE name=:n"), {"n": name}).scalar_one()
session.commit()
return ontology_id
@@ -501,3 +502,166 @@ def override_get_db():
)
assert missing.status_code == 404
test_app.dependency_overrides.clear()
+
+
+def test_zz_require_project_grant_enforces_actor_state_domain_window_and_role(access_db):
+ from types import SimpleNamespace
+
+ from app.models.user import User
+ from app.services.ontology_access import (
+ GrantDenied,
+ create_grant,
+ creator_grant,
+ require_project_grant,
+ )
+
+ Session, _ = access_db
+ owner = _insert_user(Session, "guard-owner", role="admin")
+ reader = _insert_user(Session, "guard-reader", role="editor")
+ target = _insert_user(Session, "guard-target", role="viewer")
+ ontology_id = _insert_ontology(Session, "Guard ontology", owner)
+
+ with Session() as session:
+ creator_grant(session, ontology_id, session.get(User, owner))
+ create_grant(
+ session,
+ ontology_id=ontology_id,
+ user_id=reader,
+ capabilities=["discover", "read", "edit", "publish"],
+ base_revision=0,
+ actor_id=owner,
+ )
+ reader_user = session.get(User, reader)
+ assert require_project_grant(session, reader_user, ontology_id, "edit")
+
+ reader_user.role = "viewer"
+ session.flush()
+ with pytest.raises(GrantDenied):
+ require_project_grant(session, reader_user, ontology_id, "edit")
+ assert require_project_grant(session, reader_user, ontology_id, "read")
+
+ reader_user.role = "editor"
+ session.execute(text(
+ "UPDATE ontology_project_access_grants SET valid_from=:valid_from "
+ "WHERE ontology_id=:o AND user_id=:u"
+ ), {
+ "valid_from": datetime.now(timezone.utc) + timedelta(minutes=5),
+ "o": ontology_id,
+ "u": reader,
+ })
+ with pytest.raises(GrantDenied):
+ require_project_grant(session, reader_user, ontology_id, "read")
+
+ session.execute(text(
+ "UPDATE ontology_project_access_grants SET valid_from=NULL, valid_until=:valid_until "
+ "WHERE ontology_id=:o AND user_id=:u"
+ ), {
+ "valid_until": datetime.now(timezone.utc) - timedelta(seconds=1),
+ "o": ontology_id,
+ "u": reader,
+ })
+ with pytest.raises(GrantDenied):
+ require_project_grant(session, reader_user, ontology_id, "read")
+
+ session.execute(text(
+ "UPDATE ontology_project_access_grants SET valid_until=NULL WHERE ontology_id=:o AND user_id=:u"
+ ), {"o": ontology_id, "u": reader})
+ with pytest.raises(GrantDenied):
+ require_project_grant(
+ session,
+ SimpleNamespace(
+ id=reader,
+ is_active=True,
+ role="editor",
+ security_domain_id="00000000-0000-0000-0000-000000000099",
+ ),
+ ontology_id,
+ "read",
+ )
+
+ reader_user.is_active = False
+ session.flush()
+ with pytest.raises(GrantDenied):
+ require_project_grant(session, reader_user, ontology_id, "read")
+
+ session.execute(text(
+ "UPDATE ontology_project_access_grants SET valid_until=:valid_until "
+ "WHERE ontology_id=:o AND user_id=:u"
+ ), {
+ "valid_until": datetime.now(timezone.utc) - timedelta(seconds=1),
+ "o": ontology_id,
+ "u": owner,
+ })
+ with pytest.raises(GrantDenied):
+ create_grant(
+ session,
+ ontology_id=ontology_id,
+ user_id=target,
+ capabilities=["discover", "read"],
+ base_revision=0,
+ actor_id=owner,
+ )
+
+ session.execute(text(
+ "UPDATE ontology_project_access_grants SET valid_until=NULL "
+ "WHERE ontology_id=:o AND user_id=:u"
+ ), {"o": ontology_id, "u": owner})
+ session.execute(text("UPDATE users SET role='viewer' WHERE id=:u"), {"u": owner})
+ with pytest.raises(GrantDenied):
+ create_grant(
+ session,
+ ontology_id=ontology_id,
+ user_id=target,
+ capabilities=["discover", "read"],
+ base_revision=0,
+ actor_id=owner,
+ )
+
+
+def test_zz_discoverable_ontology_ids_filter_capability_window_domain_and_role(access_db):
+ from app.models.user import User
+ from app.services.ontology_access import (
+ create_grant,
+ creator_grant,
+ discoverable_ontology_ids,
+ )
+
+ Session, _ = access_db
+ owner = _insert_user(Session, "discover-owner", role="admin")
+ viewer = _insert_user(Session, "discover-viewer", role="viewer")
+ same = _insert_ontology(Session, "Discover same", owner)
+ no_discover = _insert_ontology(Session, "Discover no capability", owner)
+ expired = _insert_ontology(Session, "Discover expired", owner)
+ future = _insert_ontology(Session, "Discover future", owner)
+
+ with Session() as session:
+ creator_grant(session, same, session.get(User, owner))
+ creator_grant(session, no_discover, session.get(User, owner))
+ creator_grant(session, expired, session.get(User, owner))
+ creator_grant(session, future, session.get(User, owner))
+ for oid, capabilities in (
+ (same, ["discover", "read"]),
+ (no_discover, ["read"]),
+ (expired, ["discover", "read"]),
+ (future, ["discover", "read"]),
+ ):
+ create_grant(
+ session,
+ ontology_id=oid,
+ user_id=viewer,
+ capabilities=capabilities,
+ base_revision=0,
+ actor_id=owner,
+ )
+ session.execute(text(
+ "UPDATE ontology_project_access_grants SET valid_until=:when "
+ "WHERE ontology_id=:o AND user_id=:u"
+ ), {"when": datetime.now(timezone.utc) - timedelta(seconds=1), "o": expired, "u": viewer})
+ session.execute(text(
+ "UPDATE ontology_project_access_grants SET valid_from=:when "
+ "WHERE ontology_id=:o AND user_id=:u"
+ ), {"when": datetime.now(timezone.utc) + timedelta(minutes=5), "o": future, "u": viewer})
+ actor = session.get(User, viewer)
+ ids = session.execute(discoverable_ontology_ids(session, actor)).scalars().all()
+
+ assert ids == [same]
diff --git a/backend/tests/agent/test_ontology_route_access.py b/backend/tests/agent/test_ontology_route_access.py
new file mode 100644
index 00000000..6eda7805
--- /dev/null
+++ b/backend/tests/agent/test_ontology_route_access.py
@@ -0,0 +1,80 @@
+"""Project grants must protect legacy and classic-v2 ontology API surfaces."""
+
+import pytest
+
+from app.main import app
+from app.routers.v2 import graph as graph_v2
+from app.routers.v2 import logic_actions as logic_actions_v2
+from app.routers.v2 import search as search_v2
+
+
+@pytest.mark.parametrize("path", [
+ "/api/v1/ontologies/{id}",
+ "/api/v1/ontologies/{id}/export?format=json",
+ "/api/v1/ontologies/{id}/entities",
+ "/api/v1/ontologies/{id}/logic",
+ "/api/v1/ontologies/{id}/actions",
+ "/api/v1/ontologies/{id}/graph",
+ "/api/v1/ontologies/{id}/files",
+ "/api/v2/ontologies/{id}/graph",
+ "/api/v2/ontologies/{id}/search/keyword?q=test",
+ "/api/v2/ontologies/{id}/logic",
+ "/api/v2/ontologies/{id}/state-machines",
+ "/api/v2/ontologies/{id}/actions",
+])
+def test_ungranted_editor_cannot_read_ontology_surfaces(client, db, ontology, editor_user, path):
+ def test_db():
+ yield db
+
+ for dependency in (graph_v2.get_db, search_v2.get_db, logic_actions_v2.get_db):
+ app.dependency_overrides[dependency] = test_db
+
+ token = client.post(
+ "/api/v1/auth/login",
+ json={"username": editor_user.username, "password": "editor123"},
+ ).json()["data"]["access_token"]
+ response = client.get(
+ path.format(id=ontology["id"]),
+ headers={"Authorization": f"Bearer {token}"},
+ )
+ assert response.status_code == 404
+
+
+@pytest.mark.parametrize("path", [
+ "/api/v1/ontologies/{id}/export?format=json",
+ "/api/v2/ontologies/{id}/search/keyword?q=test",
+ "/api/v2/ontologies/{id}/logic",
+])
+def test_creator_grant_keeps_read_routes_available(client, db, ontology, auth_headers, path):
+ def test_db():
+ yield db
+
+ for dependency in (search_v2.get_db, logic_actions_v2.get_db):
+ app.dependency_overrides[dependency] = test_db
+
+ response = client.get(path.format(id=ontology["id"]), headers=auth_headers)
+ assert response.status_code == 200
+
+
+def _editor_headers(client):
+ token = client.post(
+ "/api/v1/auth/login",
+ json={"username": "editor", "password": "editor123"},
+ ).json()["data"]["access_token"]
+ return {"Authorization": f"Bearer {token}"}
+
+
+def test_ungranted_editor_cannot_write_legacy_entities(client, ontology, editor_user):
+ response = client.post(
+ f"/api/v1/ontologies/{ontology['id']}/entities",
+ json={"name_cn": "越权实体", "name_en": "Intruder"},
+ headers=_editor_headers(client),
+ )
+ assert response.status_code == 404
+
+
+def test_overview_hides_ungranted_ontologies(client, ontology, editor_user):
+ response = client.get("/api/v1/overview/stats", headers=_editor_headers(client))
+ assert response.status_code == 200
+ data = response.json()["data"]
+ assert ontology["id"] not in [item["id"] for item in data["recent_ontologies"]]
diff --git a/backend/tests/agent/test_python_floor.py b/backend/tests/agent/test_python_floor.py
index 2ec9b4e9..595336d4 100644
--- a/backend/tests/agent/test_python_floor.py
+++ b/backend/tests/agent/test_python_floor.py
@@ -14,7 +14,7 @@
BOOTSTRAP_PATH = BACKEND_DIR / "scripts" / "bootstrap_backend.py"
PYPROJECT_PATH = BACKEND_DIR / "pyproject.toml"
UNSUPPORTED_PYTHON_VERSION = "UNSUPPORTED_PYTHON_VERSION"
-REQUIRED_PYTHON_FLOOR = ">=3.11"
+REQUIRED_PYTHON_RANGE = ">=3.11,<3.13"
def _run_entrypoint(*args, env=None):
@@ -59,10 +59,10 @@ def test_e0_py_red_contract():
if missing:
pytest.fail("RED_E0_PY: Python floor contract missing: " + ", ".join(missing))
metadata = tomllib.loads(PYPROJECT_PATH.read_text())
- if metadata.get("project", {}).get("requires-python") != REQUIRED_PYTHON_FLOOR:
+ if metadata.get("project", {}).get("requires-python") != REQUIRED_PYTHON_RANGE:
pytest.fail(
f"RED_E0_PY: pyproject.toml must declare requires-python = "
- f"{REQUIRED_PYTHON_FLOOR!r}"
+ f"{REQUIRED_PYTHON_RANGE!r}"
)
proc = _run_guard()
if proc.returncode != 0:
@@ -207,8 +207,8 @@ def test_require_supported_python_accepts_current_runtime():
assert _load_guard().require_supported_python() is True
-def test_require_supported_python_rejects_python_3_10_and_earlier():
- for major_minor in [(3, 10), (3, 9), (3, 8)]:
+def test_require_supported_python_rejects_versions_outside_supported_range():
+ for major_minor in [(3, 10), (3, 9), (3, 8), (3, 13)]:
proc = subprocess.run(
[
sys.executable,
@@ -225,7 +225,7 @@ def test_require_supported_python_rejects_python_3_10_and_earlier():
assert UNSUPPORTED_PYTHON_VERSION in message
detected = ".".join(str(part) for part in major_minor)
assert f"detected Python {detected}" in message
- assert "required >= 3.11" in message
+ assert "required >= 3.11, < 3.13" in message
def test_guard_runs_without_site_packages():
@@ -248,7 +248,7 @@ def test_guard_imports_stdlib_only():
def test_pyproject_declares_python_floor():
metadata = tomllib.loads(PYPROJECT_PATH.read_text())
- assert metadata["project"]["requires-python"] == REQUIRED_PYTHON_FLOOR
+ assert metadata["project"]["requires-python"] == REQUIRED_PYTHON_RANGE
def test_packaging_build_evaluates_and_emits_python_floor(tmp_path):
@@ -278,4 +278,4 @@ def test_packaging_build_evaluates_and_emits_python_floor(tmp_path):
assert len(installed) == 1
metadata = installed[0]["metadata"]
assert metadata["name"] == "ontexus-backend"
- assert metadata["requires_python"] == REQUIRED_PYTHON_FLOOR
+ assert set(metadata["requires_python"].split(",")) == set(REQUIRED_PYTHON_RANGE.split(","))
diff --git a/backend/tests/agent/test_simple_llm_instances.py b/backend/tests/agent/test_simple_llm_instances.py
index 7b1333f5..f5d66094 100644
--- a/backend/tests/agent/test_simple_llm_instances.py
+++ b/backend/tests/agent/test_simple_llm_instances.py
@@ -4,6 +4,7 @@
instance data attached to its concept entities (row_identity + row_data),
consistent with Pipeline/Mapping output — not just Entity/Relation rows.
"""
+import json
import os
import subprocess
import sys
@@ -162,6 +163,164 @@ def test_simple_llm_tabular_extraction_materializes_instances(schema, monkeypatc
session.close()
+def test_instance_relations_reuses_relation_from_plain_relations_array(schema, monkeypatch):
+ """Bug caught live on a 供应链 rebuild: when the LLM's own `relations` array
+ (not the instance_relations auto-create branch) is what creates the
+ concept-level Relation that an instance_relations entry then reuses via
+ relation_id_by_key, that relation must still be flushed before the
+ EntityInstanceRelation insert — otherwise it's an unflushed pending row
+ and the FK insert fails. Production's SessionLocal is autoflush=False
+ (app/database.py), unlike _worker's default-autoflush factory, so this
+ test builds its own session to match and actually catch the bug."""
+ factory = sessionmaker(bind=create_engine(_scoped_url(schema)), autoflush=False)
+ session = factory()
+ task_id = _seed(session)
+ monkeypatch.setattr("app.database.SessionLocal", factory)
+ import app.tasks.extraction as extraction_task
+
+ fake_result = {
+ "entities": [
+ {"name_cn": "供应商", "type": "Supplier", "description": "d", "properties": {}},
+ {"name_cn": "物料", "type": "Material", "description": "d", "properties": {}},
+ ],
+ "relations": [
+ {"source": "供应商", "target": "物料", "type": "supply", "confidence": 0.9},
+ ],
+ "logic_rules": [], "actions": [],
+ "instances": [
+ {"entity_type": "供应商", "name_cn": "天钢原材料有限公司", "properties": {}},
+ {"entity_type": "物料", "name_cn": "钢材", "properties": {}},
+ ],
+ "instance_relations": [
+ {"source": "天钢原材料有限公司", "target": "钢材", "type": "supply", "confidence": 0.9},
+ ],
+ }
+ monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: fake_result)
+
+ extraction_task.run_extraction(task_id)
+ try:
+ status = session.execute(text(
+ "SELECT status, error FROM extraction_tasks WHERE id = :id"
+ ), {"id": task_id}).mappings().one()
+ assert status["status"] == "completed", status["error"]
+
+ edge = session.execute(text(
+ "SELECT ei_src.row_identity AS src, ei_tgt.row_identity AS tgt "
+ "FROM entity_instance_relations eir "
+ "JOIN entity_instances ei_src ON ei_src.id = eir.source_instance_id "
+ "JOIN entity_instances ei_tgt ON ei_tgt.id = eir.target_instance_id "
+ "WHERE eir.ontology_id = 'o-llm'"
+ )).mappings().one()
+ assert edge["src"] == "天钢原材料有限公司"
+ assert edge["tgt"] == "钢材"
+ finally:
+ session.close()
+
+
+def test_instance_relations_create_concept_relation_and_instance_edge(schema, monkeypatch):
+ """A causal/interaction relation between two SPECIFIC named instances
+ (e.g. 2型糖尿病 causes 糖尿病肾病) must not be silently dropped just
+ because 'relations only connect concepts' — it auto-creates the concept-
+ level Relation it instantiates and lands as an EntityInstanceRelation
+ edge between the two instance rows. Uses an explicit autoflush=False
+ session to match production (app/database.py), not _worker's default."""
+ factory = sessionmaker(bind=create_engine(_scoped_url(schema)), autoflush=False)
+ session = factory()
+ task_id = _seed(session)
+ monkeypatch.setattr("app.database.SessionLocal", factory)
+ import app.tasks.extraction as extraction_task
+
+ fake_result = {
+ "entities": [{"name_cn": "疾病", "type": "Disease", "description": "d", "properties": {}}],
+ "relations": [], "logic_rules": [], "actions": [],
+ "instances": [
+ {"entity_type": "疾病", "name_cn": "2型糖尿病", "properties": {}},
+ {"entity_type": "疾病", "name_cn": "糖尿病肾病", "properties": {}},
+ ],
+ "instance_relations": [
+ {"source": "2型糖尿病", "target": "糖尿病肾病", "type": "causes", "confidence": 0.9},
+ ],
+ }
+ monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: fake_result)
+
+ extraction_task.run_extraction(task_id)
+ try:
+ status = session.execute(text(
+ "SELECT status, error FROM extraction_tasks WHERE id = :id"
+ ), {"id": task_id}).mappings().one()
+ assert status["status"] == "completed", status["error"]
+
+ concept_rels = session.execute(text(
+ "SELECT type FROM relations WHERE ontology_id = 'o-llm'"
+ )).mappings().all()
+ assert [r["type"] for r in concept_rels] == ["causes"]
+
+ edge = session.execute(text(
+ "SELECT ei_src.row_identity AS src, ei_tgt.row_identity AS tgt "
+ "FROM entity_instance_relations eir "
+ "JOIN entity_instances ei_src ON ei_src.id = eir.source_instance_id "
+ "JOIN entity_instances ei_tgt ON ei_tgt.id = eir.target_instance_id "
+ "WHERE eir.ontology_id = 'o-llm'"
+ )).mappings().one()
+ assert edge["src"] == "2型糖尿病"
+ assert edge["tgt"] == "糖尿病肾病"
+ finally:
+ session.close()
+
+
+def test_instance_relations_rerun_does_not_duplicate(schema, monkeypatch):
+ """Re-running extraction with the same instance_relations output must not
+ create a second EntityInstanceRelation edge or a second concept Relation.
+ Uses an explicit autoflush=False session to match production."""
+ factory = sessionmaker(bind=create_engine(_scoped_url(schema)), autoflush=False)
+ session = factory()
+ task_id_1 = _seed(session)
+ monkeypatch.setattr("app.database.SessionLocal", factory)
+ import app.tasks.extraction as extraction_task
+
+ fake_result = {
+ "entities": [{"name_cn": "疾病", "type": "Disease", "description": "d", "properties": {}}],
+ "relations": [], "logic_rules": [], "actions": [],
+ "instances": [
+ {"entity_type": "疾病", "name_cn": "2型糖尿病", "properties": {}},
+ {"entity_type": "疾病", "name_cn": "糖尿病肾病", "properties": {}},
+ ],
+ "instance_relations": [
+ {"source": "2型糖尿病", "target": "糖尿病肾病", "type": "causes", "confidence": 0.9},
+ ],
+ }
+ monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: fake_result)
+ extraction_task.run_extraction(task_id_1)
+
+ task_id_2 = str(uuid.uuid4())
+ session.execute(text(
+ "INSERT INTO extraction_tasks (id, ontology_id, prompt_id, model_id, status, parameters, progress, error, created_at, updated_at) "
+ "VALUES (:id, 'o-llm', 'p-1', 'm-1', 'queued', :params, '{}'::json, NULL, now(), now())"
+ ), {"id": task_id_2, "params": '{"model_name": "mock-extractor", "constraints": []}'})
+ session.commit()
+ monkeypatch.setattr(
+ "app.services.llm_service._call_llm",
+ lambda *a, **k: '{"clusters": []}',
+ )
+ extraction_task.run_extraction(task_id_2)
+
+ try:
+ status = session.execute(text(
+ "SELECT status, error FROM extraction_tasks WHERE id = :id"
+ ), {"id": task_id_2}).mappings().one()
+ assert status["status"] == "completed", status["error"]
+ rel_count = session.execute(text(
+ "SELECT count(*) FROM relations WHERE ontology_id = 'o-llm'"
+ )).scalar_one()
+ edge_count = session.execute(text(
+ "SELECT count(*) FROM entity_instance_relations WHERE ontology_id = 'o-llm'"
+ )).scalar_one()
+ assert rel_count == 1
+ assert edge_count == 1
+ finally:
+ session.close()
+
+
def test_single_document_near_duplicate_entities_are_resolved(schema, monkeypatch):
"""Sub-project E3 bug 1: a single-file extraction with near-duplicate
entity names (e.g. "产品研发部" vs "产品研发部门") must still go through
@@ -186,7 +345,7 @@ def test_single_document_near_duplicate_entities_are_resolved(schema, monkeypatc
)
monkeypatch.setattr(
"app.services.llm_service.resolve_entities",
- lambda entities, config, model_name: {
+ lambda entities, config, model_name, existing_entities=None: {
"产品研发部": "产品研发部",
"产品研发部门": "产品研发部",
},
@@ -237,3 +396,267 @@ def test_simple_llm_without_instances_still_succeeds(schema, monkeypatch):
)).scalar_one() == 0
finally:
session.close()
+
+
+def test_dangling_linked_entities_reference_is_dropped(schema, monkeypatch):
+ """A logic_rule/action whose linked_entities names a rule-category label
+ (e.g. "反欺诈规则") rather than a real concept entity must not carry that
+ dangling reference into the saved row — the same discipline already
+ applied to relations pointing at an entity the model never extracted."""
+ factory, session = _worker(schema)
+ task_id = _seed(session)
+ monkeypatch.setattr("app.database.SessionLocal", factory)
+ import app.tasks.extraction as extraction_task
+ monkeypatch.setattr(
+ "app.services.llm_service.extract_ontology",
+ lambda *a, **k: {
+ "entities": [{"name_cn": "设备指纹", "type": "Concept", "description": "设备指纹特征", "properties": {}}],
+ "relations": [],
+ "logic_rules": [
+ {"name_cn": "设备指纹黑名单拒绝", "function_type": "derived_property", "definition": "命中黑名单则拒绝",
+ "linked_entities": ["设备指纹", "反欺诈规则"]},
+ ],
+ "actions": [
+ {"name_cn": "标记欺诈", "rules": [], "linked_entities": ["设备指纹", "反欺诈规则"]},
+ ],
+ },
+ )
+ extraction_task.run_extraction(task_id)
+ try:
+ status = session.execute(text(
+ "SELECT status, error FROM extraction_tasks WHERE id = :id"
+ ), {"id": task_id}).mappings().one()
+ assert status["status"] == "completed", status["error"]
+ def _as_list(value):
+ return json.loads(value) if isinstance(value, str) else value
+
+ rule_linked = _as_list(session.execute(text(
+ "SELECT linked_entities FROM logic_rules WHERE ontology_id = 'o-llm' AND name_cn = '设备指纹黑名单拒绝'"
+ )).scalar_one())
+ assert rule_linked == ["设备指纹"]
+ action_linked = _as_list(session.execute(text(
+ "SELECT linked_entities FROM actions WHERE ontology_id = 'o-llm' AND name_cn = '标记欺诈'"
+ )).scalar_one())
+ assert action_linked == ["设备指纹"]
+ finally:
+ session.close()
+
+
+def test_rerun_resolves_new_surface_form_against_already_published_entity(schema, monkeypatch):
+ """Incremental-update guidance: re-running extraction on an ontology that
+ already has a published entity must not create a duplicate when the new
+ run's LLM output spells the same concept differently — it should resolve
+ to the existing entity's own name, not the other way around."""
+ factory, session = _worker(schema)
+ task_id_1 = _seed(session)
+ monkeypatch.setattr("app.database.SessionLocal", factory)
+ import app.tasks.extraction as extraction_task
+
+ monkeypatch.setattr(
+ "app.services.llm_service.extract_ontology",
+ lambda *a, **k: {
+ "entities": [{"name_cn": "客户", "type": "Concept", "description": "贷款客户", "properties": {}}],
+ "relations": [], "logic_rules": [], "actions": [],
+ },
+ )
+ extraction_task.run_extraction(task_id_1)
+
+ task_id_2 = str(uuid.uuid4())
+ session.execute(text(
+ "INSERT INTO extraction_tasks (id, ontology_id, prompt_id, model_id, status, parameters, progress, error, created_at, updated_at) "
+ "VALUES (:id, 'o-llm', 'p-1', 'm-1', 'queued', :params, '{}'::json, NULL, now(), now())"
+ ), {"id": task_id_2, "params": '{"model_name": "mock-extractor", "constraints": []}'})
+ session.commit()
+
+ monkeypatch.setattr(
+ "app.services.llm_service.extract_ontology",
+ lambda *a, **k: {
+ "entities": [{"name_cn": "客户方", "type": "Concept", "description": "本轮换了个写法", "properties": {}}],
+ "relations": [], "logic_rules": [], "actions": [],
+ },
+ )
+ monkeypatch.setattr(
+ "app.services.llm_service._call_llm",
+ lambda *a, **k: '{"clusters": [{"canonical": "客户方", "aliases": ["客户方", "客户"]}]}',
+ )
+ extraction_task.run_extraction(task_id_2)
+
+ try:
+ status = session.execute(text(
+ "SELECT status, error FROM extraction_tasks WHERE id = :id"
+ ), {"id": task_id_2}).mappings().one()
+ assert status["status"] == "completed", status["error"]
+ names = session.execute(text(
+ "SELECT name_cn FROM entities WHERE ontology_id = 'o-llm'"
+ )).scalars().all()
+ # the model's clustering call picked "客户方" as canonical, but the
+ # already-published "客户" must win — no duplicate, no rename
+ assert names == ["客户"]
+ finally:
+ session.close()
+
+
+def test_extraction_task_stores_graph_diagnostics(schema, monkeypatch):
+ """Graph diagnostics (connected components, hub entities) are computed
+ over the ontology's full saved graph and stored on the task's
+ validation_report alongside the existing P0 report."""
+ factory, session = _worker(schema)
+ task_id = _seed(session)
+ monkeypatch.setattr("app.database.SessionLocal", factory)
+ import app.tasks.extraction as extraction_task
+ monkeypatch.setattr(
+ "app.services.llm_service.extract_ontology",
+ lambda *a, **k: {
+ "entities": [
+ {"name_cn": "借款人", "type": "Concept", "description": "d", "properties": {}},
+ {"name_cn": "贷款", "type": "Concept", "description": "d", "properties": {}},
+ {"name_cn": "孤立实体", "type": "Concept", "description": "d", "properties": {}},
+ ],
+ "relations": [{"source": "借款人", "target": "贷款", "type": "APPLIES_FOR", "confidence": 0.9}],
+ "logic_rules": [], "actions": [],
+ },
+ )
+ extraction_task.run_extraction(task_id)
+ try:
+ report = session.execute(text(
+ "SELECT validation_report FROM extraction_tasks WHERE id = :id"
+ ), {"id": task_id}).scalar_one()
+ diagnostics = json.loads(report)["graph_diagnostics"] if isinstance(report, str) else report["graph_diagnostics"]
+ assert diagnostics["entity_count"] == 3
+ assert diagnostics["edge_count"] == 1
+ assert diagnostics["connected_components"] == 2 # {借款人,贷款} + {孤立实体}
+ assert diagnostics["isolated_entity_count"] == 1
+ assert diagnostics["isolated_entities"] == ["孤立实体"]
+ finally:
+ session.close()
+
+
+def test_graph_diagnostics_sees_this_runs_own_relations_under_autoflush_off(schema, monkeypatch):
+ """Production's SessionLocal is autoflush=False (app/database.py), unlike
+ the _worker fixture's default-autoflush factory used above. The
+ diagnostics query must still see this run's own newly-added entities and
+ relations rather than only whatever was committed before this run began."""
+ factory = sessionmaker(bind=create_engine(_scoped_url(schema)), autoflush=False)
+ session = factory()
+ task_id = _seed(session)
+ monkeypatch.setattr("app.database.SessionLocal", factory)
+ import app.tasks.extraction as extraction_task
+ monkeypatch.setattr(
+ "app.services.llm_service.extract_ontology",
+ lambda *a, **k: {
+ "entities": [
+ {"name_cn": "借款人", "type": "Concept", "description": "d", "properties": {}},
+ {"name_cn": "贷款", "type": "Concept", "description": "d", "properties": {}},
+ ],
+ "relations": [{"source": "借款人", "target": "贷款", "type": "APPLIES_FOR", "confidence": 0.9}],
+ "logic_rules": [], "actions": [],
+ },
+ )
+ extraction_task.run_extraction(task_id)
+ try:
+ report = session.execute(text(
+ "SELECT validation_report FROM extraction_tasks WHERE id = :id"
+ ), {"id": task_id}).scalar_one()
+ diagnostics = json.loads(report)["graph_diagnostics"] if isinstance(report, str) else report["graph_diagnostics"]
+ assert diagnostics["edge_count"] == 1
+ assert diagnostics["connected_components"] == 1
+ finally:
+ session.close()
+
+
+_HUB_RESULT = {
+ "entities": [
+ {"name_cn": "借款人", "type": "Concept", "description": "贷款申请人", "properties": {}},
+ {"name_cn": "贷款", "type": "Concept", "description": "d", "properties": {}},
+ {"name_cn": "授信额度", "type": "Concept", "description": "d", "properties": {}},
+ {"name_cn": "信用评分", "type": "Concept", "description": "d", "properties": {}},
+ ],
+ "relations": [
+ {"source": "借款人", "target": "贷款", "type": "APPLIES_FOR", "confidence": 0.9},
+ {"source": "借款人", "target": "授信额度", "type": "HAS", "confidence": 0.9},
+ {"source": "借款人", "target": "信用评分", "type": "HAS", "confidence": 0.9},
+ ],
+ "logic_rules": [], "actions": [],
+}
+
+
+def test_hub_entity_gets_summarized(schema, monkeypatch):
+ """A degree-3+ entity (playbook Section V.A hub threshold) gets a
+ synthesized profile stored on its properties; low-degree entities do not."""
+ factory, session = _worker(schema)
+ task_id = _seed(session)
+ monkeypatch.setattr("app.database.SessionLocal", factory)
+ import app.tasks.extraction as extraction_task
+ monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: _HUB_RESULT)
+ monkeypatch.setattr(
+ "app.services.llm_service._call_llm",
+ lambda *a, **k: '{"summary": "借款人是核心概念。", "key_facts": ["f1"], '
+ '"time_range": {"start": "2026", "end": "ongoing"}}',
+ )
+ extraction_task.run_extraction(task_id)
+ try:
+ status = session.execute(text(
+ "SELECT status, error FROM extraction_tasks WHERE id = :id"
+ ), {"id": task_id}).mappings().one()
+ assert status["status"] == "completed", status["error"]
+ rows = session.execute(text(
+ "SELECT name_cn, properties FROM entities WHERE ontology_id = 'o-llm'"
+ )).mappings().all()
+ props_by_name = {r["name_cn"]: (json.loads(r["properties"]) if isinstance(r["properties"], str) else r["properties"]) for r in rows}
+ assert props_by_name["借款人"]["summary"] == "借款人是核心概念。"
+ assert props_by_name["借款人"]["summary_degree"] == 3
+ # a degree-1 entity is not summarized
+ assert "summary" not in props_by_name["贷款"]
+ finally:
+ session.close()
+
+
+def test_hub_entity_summary_skipped_when_degree_unchanged_on_rerun(schema, monkeypatch):
+ """Re-running with the same graph shape must not re-call the summarizer —
+ only a changed source-document set (reflected here as a changed degree)
+ should trigger re-summarization."""
+ factory, session = _worker(schema)
+ task_id_1 = _seed(session)
+ monkeypatch.setattr("app.database.SessionLocal", factory)
+ import app.tasks.extraction as extraction_task
+ monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: _HUB_RESULT)
+ # resolve_entities isn't exercised on a from-scratch run (nothing existing
+ # to resolve against yet), so mocking summarize_entity alone is enough
+ # here and avoids also stubbing out the (unrelated) resolution LLM call
+ monkeypatch.setattr(
+ "app.services.llm_service.summarize_entity",
+ lambda *a, **k: {"summary": "first pass", "key_facts": [], "time_range": {"start": "unknown", "end": "unknown"}},
+ )
+ extraction_task.run_extraction(task_id_1)
+
+ task_id_2 = str(uuid.uuid4())
+ session.execute(text(
+ "INSERT INTO extraction_tasks (id, ontology_id, prompt_id, model_id, status, parameters, progress, error, created_at, updated_at) "
+ "VALUES (:id, 'o-llm', 'p-1', 'm-1', 'queued', :params, '{}'::json, NULL, now(), now())"
+ ), {"id": task_id_2, "params": '{"model_name": "mock-extractor", "constraints": []}'})
+ session.commit()
+
+ monkeypatch.setattr("app.services.llm_service.extract_ontology", lambda *a, **k: _HUB_RESULT)
+ # resolve_entities still runs (4 new entities of the same type) and needs
+ # a harmless response so it doesn't attempt a real network call; the
+ # assertion this test cares about is that summarize_entity is skipped
+ monkeypatch.setattr("app.services.llm_service._call_llm", lambda *a, **k: '{"clusters": []}')
+ monkeypatch.setattr(
+ "app.services.llm_service.summarize_entity",
+ lambda *a, **k: (_ for _ in ()).throw(AssertionError("summarizer should not be called again")),
+ )
+ extraction_task.run_extraction(task_id_2)
+
+ try:
+ status = session.execute(text(
+ "SELECT status, error FROM extraction_tasks WHERE id = :id"
+ ), {"id": task_id_2}).mappings().one()
+ assert status["status"] == "completed", status["error"]
+ properties = session.execute(text(
+ "SELECT properties FROM entities WHERE ontology_id = 'o-llm' AND name_cn = '借款人'"
+ )).scalar_one()
+ properties = json.loads(properties) if isinstance(properties, str) else properties
+ # untouched from the first run — the summarizer was never invoked again
+ assert properties["summary"] == "first pass"
+ finally:
+ session.close()
diff --git a/backend/tests/evals/test_document_extraction_validators.py b/backend/tests/evals/test_document_extraction_validators.py
index 46be2409..4229f6c0 100644
--- a/backend/tests/evals/test_document_extraction_validators.py
+++ b/backend/tests/evals/test_document_extraction_validators.py
@@ -3,7 +3,9 @@
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
-from evals.document_extraction.validators import keyword_recall, dedup_gate, instance_leakage_gate
+from evals.document_extraction.validators import (
+ keyword_recall, dedup_gate, instance_leakage_gate, rule_leakage_gate, precision_recall,
+)
SAMPLE_RESULT = {
"entities": [
@@ -101,3 +103,101 @@ def test_instance_leakage_gate_clean_for_concept_names():
"relations": [], "logic_rules": [], "actions": [], "instances": [],
}
assert instance_leakage_gate(result) == []
+
+
+def test_instance_leakage_gate_flags_enum_value_entity_names():
+ result = {
+ "entities": [
+ {"name_cn": "信用分层", "description": "概念实体"},
+ {"name_cn": "A层", "description": "leaked enum value"},
+ {"name_cn": "M2+", "description": "leaked enum value"},
+ {"name_cn": "D轮融资", "description": "leaked enum value"},
+ ],
+ "relations": [], "logic_rules": [], "actions": [], "instances": [],
+ }
+ findings = instance_leakage_gate(result)
+ flagged_names = {f["name"] for f in findings}
+ assert flagged_names == {"A层", "M2+", "D轮融资"}
+
+
+def test_rule_leakage_gate_flags_business_rule_entity_names():
+ result = {
+ "entities": [
+ {"name_cn": "借款人", "description": "概念实体"},
+ {"name_cn": "促销定价规则", "description": "leaked business rule"},
+ {"name_cn": "风险准入政策", "description": "leaked business rule"},
+ ],
+ "relations": [], "logic_rules": [], "actions": [], "instances": [],
+ }
+ findings = rule_leakage_gate(result)
+ flagged_names = {f["name"] for f in findings}
+ assert flagged_names == {"促销定价规则", "风险准入政策"}
+
+
+def test_rule_leakage_gate_clean_for_concept_names():
+ result = {
+ "entities": [
+ {"name_cn": "借款人", "description": "概念实体"},
+ {"name_cn": "信用分层", "description": "概念实体"},
+ ],
+ "relations": [], "logic_rules": [], "actions": [], "instances": [],
+ }
+ assert rule_leakage_gate(result) == []
+
+
+def test_precision_recall_perfect_match():
+ gold = ["借款人", "信用分层"]
+ result = {"entities": [{"name_cn": "借款人"}, {"name_cn": "信用分层"}]}
+ pr = precision_recall(gold, result)
+ assert pr["precision"] == 1.0
+ assert pr["recall"] == 1.0
+ assert pr["f1"] == 1.0
+ assert pr["false_positive"] == 0
+ assert pr["false_negative"] == 0
+
+
+def test_precision_recall_counts_extra_extracted_entities_as_false_positives():
+ gold = ["借款人"]
+ result = {"entities": [{"name_cn": "借款人"}, {"name_cn": "促销定价规则"}]}
+ pr = precision_recall(gold, result)
+ assert pr["true_positive"] == 1
+ assert pr["false_positive"] == 1
+ assert pr["recall"] == 1.0
+ assert pr["precision"] == 0.5
+ assert pr["unmatched_extracted_entities"] == ["促销定价规则"]
+
+
+def test_precision_recall_counts_missing_gold_entities_as_false_negatives():
+ gold = ["借款人", "授信额度"]
+ result = {"entities": [{"name_cn": "借款人"}]}
+ pr = precision_recall(gold, result)
+ assert pr["true_positive"] == 1
+ assert pr["false_negative"] == 1
+ assert pr["precision"] == 1.0
+ assert pr["recall"] == 0.5
+ assert pr["missed_gold_entities"] == ["授信额度"]
+
+
+def test_precision_recall_fuzzy_matches_minor_surface_form_differences():
+ gold = ["产品研发部"]
+ result = {"entities": [{"name_cn": "产品研发部门"}]}
+ pr = precision_recall(gold, result)
+ assert pr["true_positive"] == 1
+ assert pr["missed_gold_entities"] == []
+
+
+def test_precision_recall_does_not_double_count_one_extracted_entity_for_two_gold_entities():
+ """Each extracted entity can satisfy at most one gold entity — a single
+ lucky match must not inflate recall for an unrelated second gold entity."""
+ gold = ["借款人", "还款人"]
+ result = {"entities": [{"name_cn": "借款人"}]}
+ pr = precision_recall(gold, result)
+ assert pr["true_positive"] == 1
+ assert pr["false_negative"] == 1
+
+
+def test_precision_recall_empty_gold_is_perfect_precision_no_recall_penalty():
+ pr = precision_recall([], {"entities": [{"name_cn": "任意实体"}]})
+ assert pr["recall"] == 1.0 # nothing was required
+ assert pr["false_positive"] == 1
+ assert pr["precision"] == 0.0
diff --git a/backend/tests/runtime/test_snapshot_freshness.py b/backend/tests/runtime/test_snapshot_freshness.py
index a8240004..3f83e6db 100644
--- a/backend/tests/runtime/test_snapshot_freshness.py
+++ b/backend/tests/runtime/test_snapshot_freshness.py
@@ -40,14 +40,9 @@
from app.services.runtime.snapshots import get_snapshot, materialize_refresh_snapshot
from app.services.v2.incremental.operations import get_last_successful_refresh_run
-# Captured once at import so every case in this module reasons about the
-# same instant: `test_fresh_snapshot_exposes_cursor_lag_and_lineage` needs
-# an exact lag match against a `compute_snapshot_freshness(..., now=FIXED_NOW)`
-# call, while `evaluate_fixture_runtime_freshness` exercises the real
-# `RuntimeService` (which reads real wall-clock time internally) moments
-# later in the same test process — negligible drift against this module's
-# hour/day-scale lag buckets.
-FIXED_NOW = datetime.now(timezone.utc)
+def _now() -> datetime:
+ """Capture wall-clock time per fixture/test, not once at module import."""
+ return datetime.now(timezone.utc)
DOMAIN = "00000000-0000-0000-0000-0000000000f0"
ONTOLOGY_ID = "ontology-freshness-001"
@@ -167,17 +162,18 @@ def _seed_completed_run(db, *, dataset_version_id: str, pipeline_run_id: str, re
db.commit()
-def materialize_refresh_fixture(db, *, refresh_run_id: str):
+def materialize_refresh_fixture(db, *, refresh_run_id: str, now: datetime | None = None):
"""Seed one governed dataset/pipeline pair plus the named `RefreshRun`
and materialize the snapshot it backs.
`refresh_run_id` selects the case shape: a plain success id gets a
- fresh cursor/lag anchored to `FIXED_NOW`; "soft-stale"/"hard-stale" get
+ fresh cursor/lag anchored to the supplied/current test time; "soft-stale"/"hard-stale" get
a cursor whose lag already exceeds the shared default policy's
soft/hard thresholds; "unknown-cursor" gets a successful run with no
recorded cursor at all.
"""
release = _ensure_release(db)
+ now = now or _now()
dataset_version_id = f"dv-{refresh_run_id}"
# "refresh-success-001" is this module's canonical fresh case, matching
# the exact `pipeline_run_ids == ["pipeline-run-001"]` assertion below.
@@ -198,7 +194,7 @@ def materialize_refresh_fixture(db, *, refresh_run_id: str):
)
else:
lag = _CASE_LAG_SECONDS[refresh_run_id]
- observed_at = FIXED_NOW - timedelta(seconds=lag)
+ observed_at = now - timedelta(seconds=lag)
run = RefreshRun(
id=refresh_run_id, source_id="source-freshness-001", resource="orders",
policy="batch", config_version=1, cursor_contract="watermark_primary_key",
@@ -296,9 +292,10 @@ def get_action_plan(db, plan_id):
def test_fresh_snapshot_exposes_cursor_lag_and_lineage(db):
- snapshot = materialize_refresh_fixture(db, refresh_run_id="refresh-success-001")
+ now = _now()
+ snapshot = materialize_refresh_fixture(db, refresh_run_id="refresh-success-001", now=now)
view = compute_snapshot_freshness(
- snapshot, now=FIXED_NOW, policy=FreshnessPolicy(max_lag_seconds=300, stale_action="deny"),
+ snapshot, now=now, policy=FreshnessPolicy(max_lag_seconds=300, stale_action="deny"),
)
assert view.state == "fresh"
assert view.lag_seconds == 120
@@ -328,7 +325,7 @@ def test_unrelated_refresh_run_cannot_stamp_freshness_onto_unconnected_dataset_v
idempotency_key="idem-refresh-unrelated-001",
cursor_after_json={
"watermark": None, "primary_key": "999", "opaque_value": None,
- "observed_at": FIXED_NOW.isoformat(),
+ "observed_at": _now().isoformat(),
},
fencing_token=1, lag_seconds=5, pipeline_run_id="pr-unrelated-001",
)
@@ -429,6 +426,7 @@ def test_investigate_succeeds_with_real_production_shaped_refresh_provenance(db,
refresh_run_id = "refresh-provenance-shape-001"
dataset_version_id = f"dv-{refresh_run_id}"
pipeline_run_id = f"pr-{refresh_run_id}"
+ now = _now()
# The exact dict shape production code builds — no source_type/locator/
# content_hash key anywhere (see polling.py:630-638, event_ingest.py:668-678).
@@ -439,7 +437,7 @@ def test_investigate_succeeds_with_real_production_shaped_refresh_provenance(db,
"config_version": 1,
"cursor_contract": "watermark_primary_key",
"cursor_outcome": "advanced",
- "source_observed_at": (FIXED_NOW - timedelta(seconds=60)).isoformat(),
+ "source_observed_at": (now - timedelta(seconds=60)).isoformat(),
}
pipeline = Pipeline(id=f"pipeline-{pipeline_run_id}", name=pipeline_run_id, spec={})
@@ -468,7 +466,7 @@ def test_investigate_succeeds_with_real_production_shaped_refresh_provenance(db,
idempotency_key=f"idem-{refresh_run_id}",
cursor_after_json={
"watermark": None, "primary_key": "100", "opaque_value": None,
- "observed_at": (FIXED_NOW - timedelta(seconds=60)).isoformat(),
+ "observed_at": (now - timedelta(seconds=60)).isoformat(),
},
fencing_token=1, lag_seconds=60, pipeline_run_id=pipeline_run_id,
source_provenance=dict(provenance),
diff --git a/backend/tests/test_extraction_chunking.py b/backend/tests/test_extraction_chunking.py
new file mode 100644
index 00000000..1636aeeb
--- /dev/null
+++ b/backend/tests/test_extraction_chunking.py
@@ -0,0 +1,50 @@
+"""Long-document chunking (graph-engineering playbook, scaling guidance):
+a document over the size threshold is chunked at section boundaries with a
+trailing overlap, so a single extraction call never has to hold the whole
+document — short documents (the common case) pass through untouched.
+"""
+from app.tasks.extraction import _chunk_document, _split_sections
+
+
+def test_short_document_is_not_chunked():
+ md = "# 标题\n\n短文档内容。"
+ assert _chunk_document(md, max_chars=1000) == [md]
+
+
+def test_long_document_splits_at_section_headers():
+ md = (
+ "# 第一节\n\n" + "甲" * 40 + "\n\n"
+ "# 第二节\n\n" + "乙" * 40 + "\n\n"
+ "# 第三节\n\n" + "丙" * 40
+ )
+ chunks = _chunk_document(md, max_chars=60, overlap_chars=0)
+ assert len(chunks) > 1
+ assert "甲" * 40 in "".join(chunks)
+ assert "乙" * 40 in "".join(chunks)
+ assert "丙" * 40 in "".join(chunks)
+ # each section stays whole within its chunk, never split mid-section
+ for section in ("# 第一节", "# 第二节", "# 第三节"):
+ assert sum(section in c for c in chunks) == 1
+
+
+def test_chunk_boundary_repeats_overlap_from_previous_chunk():
+ md = (
+ "# 第一节\n\n" + "甲" * 40 + "\n\n"
+ "# 第二节\n\n" + "乙" * 40
+ )
+ chunks = _chunk_document(md, max_chars=50, overlap_chars=10)
+ assert len(chunks) == 2
+ # the tail of chunk 1 reappears at the start of chunk 2
+ assert chunks[0][-10:] in chunks[1]
+
+
+def test_document_without_headers_falls_back_to_paragraph_splitting():
+ md = "\n\n".join(f"段落{i}" * 20 for i in range(5))
+ chunks = _chunk_document(md, max_chars=60, overlap_chars=0)
+ assert len(chunks) > 1
+ for i in range(5):
+ assert sum(f"段落{i}" * 20 in c for c in chunks) == 1
+
+
+def test_split_sections_single_section_when_no_headers_and_short():
+ assert _split_sections("一段没有标题的短文本") == ["一段没有标题的短文本"]
diff --git a/backend/tests/test_extraction_model_selection.py b/backend/tests/test_extraction_model_selection.py
new file mode 100644
index 00000000..efaf68fc
--- /dev/null
+++ b/backend/tests/test_extraction_model_selection.py
@@ -0,0 +1,35 @@
+"""Tiered model selection (graph-engineering playbook, model selection):
+resolution/relation-inference weigh conflicting evidence and should use a
+stronger sibling model when the user picked a fast/cheap variant for
+extraction — with zero new user-facing setting, reading only what's already
+configured in ModelConfig.models.
+"""
+from app.tasks.extraction import _select_reasoning_model
+
+
+def test_upgrades_from_fast_variant_to_stronger_sibling():
+ assert _select_reasoning_model(
+ "deepseek-flash", ["deepseek-v4-pro", "deepseek-flash"]
+ ) == "deepseek-v4-pro"
+
+
+def test_recognizes_common_fast_naming_conventions():
+ assert _select_reasoning_model("gpt-4o-mini", ["gpt-4o", "gpt-4o-mini"]) == "gpt-4o"
+ assert _select_reasoning_model("claude-haiku", ["claude-sonnet", "claude-haiku"]) == "claude-sonnet"
+
+
+def test_falls_back_to_same_model_when_not_a_fast_variant():
+ """The user already picked the strong model — never downgrade it."""
+ assert _select_reasoning_model("deepseek-v4-pro", ["deepseek-v4-pro", "deepseek-flash"]) == "deepseek-v4-pro"
+
+
+def test_falls_back_to_same_model_when_no_stronger_sibling_configured():
+ assert _select_reasoning_model("deepseek-flash", ["deepseek-flash"]) == "deepseek-flash"
+ assert _select_reasoning_model("deepseek-flash", []) == "deepseek-flash"
+ assert _select_reasoning_model("deepseek-flash", None) == "deepseek-flash"
+
+
+def test_does_not_pick_another_fast_variant_as_the_upgrade():
+ assert _select_reasoning_model(
+ "gpt-4o-mini", ["gpt-4o-mini", "claude-haiku"]
+ ) == "gpt-4o-mini"
diff --git a/backend/tests/test_graph_diagnostics.py b/backend/tests/test_graph_diagnostics.py
new file mode 100644
index 00000000..6d72185b
--- /dev/null
+++ b/backend/tests/test_graph_diagnostics.py
@@ -0,0 +1,61 @@
+"""Graph connectivity diagnostics (graph-engineering playbook, Section V.E):
+connected components, edge/node density, and hub nodes as a health signal
+for whether entity resolution actually merged what it should have.
+"""
+from app.services.graph_diagnostics import compute_graph_diagnostics
+
+
+def _entities(*names):
+ return [{"id": n, "name_cn": n} for n in names]
+
+
+def test_single_connected_component_when_fully_linked():
+ entities = _entities("借款人", "贷款", "还款记录")
+ edges = [("借款人", "贷款"), ("贷款", "还款记录")]
+ diag = compute_graph_diagnostics(entities, edges)
+ assert diag["connected_components"] == 1
+ assert diag["largest_component_size"] == 3
+ assert diag["isolated_entity_count"] == 0
+
+
+def test_fragmented_islands_are_counted_separately():
+ entities = _entities("A", "B", "C", "D")
+ edges = [("A", "B"), ("C", "D")]
+ diag = compute_graph_diagnostics(entities, edges)
+ assert diag["connected_components"] == 2
+ assert diag["largest_component_size"] == 2
+
+
+def test_isolated_entities_are_zero_degree_and_reported():
+ entities = _entities("A", "B", "孤立实体")
+ edges = [("A", "B")]
+ diag = compute_graph_diagnostics(entities, edges)
+ assert diag["isolated_entity_count"] == 1
+ assert diag["isolated_entities"] == ["孤立实体"]
+
+
+def test_edge_node_ratio_and_hub_ranking():
+ entities = _entities("hub", "a", "b", "c")
+ edges = [("hub", "a"), ("hub", "b"), ("hub", "c")]
+ diag = compute_graph_diagnostics(entities, edges)
+ assert diag["entity_count"] == 4
+ assert diag["edge_count"] == 3
+ assert diag["edge_node_ratio"] == 0.75
+ assert diag["hub_entities"][0] == {"name_cn": "hub", "degree": 3}
+
+
+def test_dangling_edge_endpoints_are_ignored():
+ """An edge referencing an entity id not in the ontology (a data quality
+ issue caught elsewhere by P0 validation) must not crash diagnostics."""
+ entities = _entities("A", "B")
+ edges = [("A", "B"), ("A", "不存在的实体")]
+ diag = compute_graph_diagnostics(entities, edges)
+ assert diag["edge_count"] == 1
+ assert diag["connected_components"] == 1
+
+
+def test_empty_ontology_does_not_divide_by_zero():
+ diag = compute_graph_diagnostics([], [])
+ assert diag["entity_count"] == 0
+ assert diag["edge_node_ratio"] == 0.0
+ assert diag["connected_components"] == 0
diff --git a/backend/tests/test_graph_files_regressions.py b/backend/tests/test_graph_files_regressions.py
new file mode 100644
index 00000000..1c093905
--- /dev/null
+++ b/backend/tests/test_graph_files_regressions.py
@@ -0,0 +1,95 @@
+import io
+import os
+
+import pytest
+
+from app.config import settings
+
+
+def test_create_relation_rejects_missing_entity_key_without_500(client, auth_headers, ontology):
+ response = client.post(
+ f"/api/v1/ontologies/{ontology['id']}/graph/relations",
+ json={"source_entity": "missing-target"},
+ headers=auth_headers,
+ )
+
+ assert response.status_code == 422
+
+
+def test_create_relation_rejects_entity_from_another_ontology(client, auth_headers, ontology):
+ first = client.post(
+ f"/api/v1/ontologies/{ontology['id']}/entities",
+ json={"name_cn": "First"},
+ headers=auth_headers,
+ ).json()["data"]
+ other_ontology = client.post(
+ "/api/v1/ontologies",
+ json={"name": "Other ontology", "domain": "供应链"},
+ headers=auth_headers,
+ ).json()["data"]
+ second = client.post(
+ f"/api/v1/ontologies/{other_ontology['id']}/entities",
+ json={"name_cn": "Second"},
+ headers=auth_headers,
+ ).json()["data"]
+
+ response = client.post(
+ f"/api/v1/ontologies/{ontology['id']}/graph/relations",
+ json={"source_entity": first["id"], "target_entity": second["id"]},
+ headers=auth_headers,
+ )
+
+ assert response.status_code == 422
+
+
+def test_upload_conversion_failure_removes_partial_file(client, auth_headers, ontology, tmp_path, monkeypatch):
+ monkeypatch.setattr(settings, "uploads_dir", str(tmp_path))
+
+ class FailedConversion:
+ ok = False
+ content = None
+ error = "conversion failed"
+
+ monkeypatch.setattr("app.routers.files.convert_document", lambda *_args: FailedConversion())
+ response = client.post(
+ f"/api/v1/ontologies/{ontology['id']}/files",
+ files={"file": ("bad.txt", io.BytesIO(b"not usable"), "text/plain")},
+ headers=auth_headers,
+ )
+
+ assert response.status_code == 422
+ assert list(tmp_path.rglob("*")) == [tmp_path / ontology["id"]]
+
+
+def test_delete_file_does_not_remove_disk_file_before_commit(
+ client, auth_headers, ontology, admin_user, db, tmp_path, monkeypatch,
+):
+ monkeypatch.setattr(settings, "uploads_dir", str(tmp_path))
+ uploaded = client.post(
+ f"/api/v1/ontologies/{ontology['id']}/files",
+ files={"file": ("ok.txt", io.BytesIO(b"hello"), "text/plain")},
+ headers=auth_headers,
+ )
+ assert uploaded.status_code == 201
+ # The public response intentionally omits file_path; discover it from the
+ # temporary ontology directory instead.
+ paths = [path for path in (tmp_path / ontology["id"]).iterdir() if path.is_file()]
+ assert len(paths) == 1
+ file_path = paths[0]
+
+ from app.routers import files as files_router
+
+ original_commit = type(db).commit
+
+ def failing_commit(self):
+ raise RuntimeError("commit failed")
+
+ monkeypatch.setattr(type(db), "commit", failing_commit)
+ try:
+ with pytest.raises(RuntimeError):
+ files_router.delete_file(
+ ontology["id"], uploaded.json()["data"]["id"], db, current_user=admin_user,
+ )
+ finally:
+ monkeypatch.setattr(type(db), "commit", original_commit)
+ assert os.path.exists(file_path)
diff --git a/backend/tests/test_ontologies.py b/backend/tests/test_ontologies.py
index 754a9506..03680d76 100644
--- a/backend/tests/test_ontologies.py
+++ b/backend/tests/test_ontologies.py
@@ -1,3 +1,51 @@
+def test_delete_ontology_cleans_up_neo4j_and_chroma(client, auth_headers, monkeypatch):
+ """Deleting an ontology must also remove its Neo4j nodes/relations and its
+ Chroma vector collection — those stores have no other owner once the
+ Postgres row is gone, and previously nothing in the delete path ever
+ called their (already-implemented) delete methods."""
+ calls = {}
+
+ class FakeNeo4j:
+ def delete_by_ontology(self, ontology_id):
+ calls["neo4j_ontology_id"] = ontology_id
+ return 3
+
+ class FakeChroma:
+ def delete_collection(self, name):
+ calls["chroma_collection"] = name
+ return True
+
+ monkeypatch.setattr("app.services.v2.graph.neo4j_service.Neo4jService", FakeNeo4j)
+ monkeypatch.setattr("app.services.v2.vector.chroma_service.ChromaService", FakeChroma)
+
+ r = client.post("/api/v1/ontologies", json={"name": "CleanupTest", "domain": "财务"}, headers=auth_headers)
+ oid = r.json()["data"]["id"]
+ r2 = client.delete(f"/api/v1/ontologies/{oid}", headers=auth_headers)
+ assert r2.status_code == 204
+ assert calls["neo4j_ontology_id"] == oid
+ assert calls["chroma_collection"] == f"ontology_{oid}"
+
+
+def test_delete_ontology_survives_neo4j_and_chroma_failure(client, auth_headers, monkeypatch):
+ """External-store cleanup is best-effort: a Postgres delete that already
+ committed must still return 204 even if Neo4j/Chroma cleanup blows up."""
+ class BrokenNeo4j:
+ def delete_by_ontology(self, ontology_id):
+ raise ConnectionError("neo4j unreachable")
+
+ class BrokenChroma:
+ def delete_collection(self, name):
+ raise ConnectionError("chroma unreachable")
+
+ monkeypatch.setattr("app.services.v2.graph.neo4j_service.Neo4jService", BrokenNeo4j)
+ monkeypatch.setattr("app.services.v2.vector.chroma_service.ChromaService", BrokenChroma)
+
+ r = client.post("/api/v1/ontologies", json={"name": "CleanupFailTest", "domain": "财务"}, headers=auth_headers)
+ oid = r.json()["data"]["id"]
+ r2 = client.delete(f"/api/v1/ontologies/{oid}", headers=auth_headers)
+ assert r2.status_code == 204
+
+
def test_create_ontology(client, auth_headers):
r = client.post("/api/v1/ontologies",
json={"name": "供应链测试", "domain": "供应链"},
@@ -31,6 +79,15 @@ def test_get_ontology(client, auth_headers):
assert r2.status_code == 200
assert r2.json()["data"]["name"] == "GetTest"
+def test_get_ontology_exposes_is_dirty(client, auth_headers, db):
+ from app.models.ontology import OntologyProject
+ r = client.post("/api/v1/ontologies", json={"name": "DirtyTest", "domain": "财务"}, headers=auth_headers)
+ oid = r.json()["data"]["id"]
+ assert client.get(f"/api/v1/ontologies/{oid}", headers=auth_headers).json()["data"]["is_dirty"] is False
+ db.query(OntologyProject).filter(OntologyProject.id == oid).update({"is_dirty": True})
+ db.commit()
+ assert client.get(f"/api/v1/ontologies/{oid}", headers=auth_headers).json()["data"]["is_dirty"] is True
+
def test_delete_ontology(client, auth_headers):
r = client.post("/api/v1/ontologies", json={"name": "Del", "domain": "财务"}, headers=auth_headers)
oid = r.json()["data"]["id"]
diff --git a/backend/tests/test_ontology_deletion_with_instances.py b/backend/tests/test_ontology_deletion_with_instances.py
new file mode 100644
index 00000000..ecc75074
--- /dev/null
+++ b/backend/tests/test_ontology_deletion_with_instances.py
@@ -0,0 +1,107 @@
+"""Deleting an ontology that has entity_instance_relations data must not hit
+ONTOLOGY_HAS_DEPENDENT_DATA — the AFTER DELETE trigger on entity_instance_relations
+(and entity_instances) re-emits agent_index_outbox rows, which must be swept up
+by cleaning agent_index_outbox LAST, not before those tables. Needs a real
+Postgres schema (SQLite has none of these triggers)."""
+import os
+import uuid
+from urllib.parse import quote
+
+import pytest
+from sqlalchemy import create_engine, text
+from sqlalchemy.orm import sessionmaker
+
+from app.routers.ontologies import delete_ontology
+
+TEST_DATABASE_URL = os.environ.get("TEST_DATABASE_URL")
+DEFAULT_DOMAIN = "00000000-0000-0000-0000-000000000001"
+
+
+def _scoped_url(schema: str) -> str:
+ return f"{TEST_DATABASE_URL}?options={quote(f'-csearch_path={schema},public', safe='-=,')}"
+
+
+@pytest.fixture
+def schema():
+ if not TEST_DATABASE_URL:
+ pytest.skip("TEST_DATABASE_URL required")
+ schema = "ontology_delete_" + uuid.uuid4().hex
+ engine = create_engine(TEST_DATABASE_URL)
+ with engine.begin() as connection:
+ connection.execute(text(f'CREATE SCHEMA "{schema}"'))
+ import subprocess
+ import sys
+ from pathlib import Path
+ backend_dir = Path(__file__).resolve().parents[1]
+ result = subprocess.run(
+ [sys.executable, "scripts/run_migrations.py", "upgrade", "head"],
+ cwd=backend_dir, env=dict(os.environ, DATABASE_URL=_scoped_url(schema)),
+ capture_output=True, text=True,
+ )
+ assert result.returncode == 0, result.stderr
+ yield schema
+ with engine.begin() as connection:
+ connection.execute(text(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE'))
+ engine.dispose()
+
+
+def test_delete_ontology_with_instance_relations_does_not_hit_dependent_data(schema, monkeypatch):
+ engine = create_engine(_scoped_url(schema))
+ session = sessionmaker(bind=engine)()
+ try:
+ _run_delete_scenario(session)
+ finally:
+ # an open transaction would block the fixture's DROP SCHEMA forever
+ session.close()
+ engine.dispose()
+
+
+def _run_delete_scenario(session):
+ from app.models.user import User
+ from app.services.ontology_access import backfill_creator_grants
+
+ oid, u = "o-del-1", "u-del-1"
+ session.execute(text(
+ "INSERT INTO users (id,username,email,password_hash,role,is_active,security_domain_id,created_at,updated_at) "
+ "VALUES (:u,'d','d@t.com','h','admin',true,:d,now(),now())"
+ ), {"u": u, "d": DEFAULT_DOMAIN})
+ session.execute(text(
+ "INSERT INTO ontology_projects (id,name,domain,version,status,created_by,created_at,updated_at,security_domain_id,working_revision) "
+ "VALUES (:o,'del-test','credit','v1','created',:u,now(),now(),:d,1)"
+ ), {"o": oid, "u": u, "d": DEFAULT_DOMAIN})
+ session.execute(text(
+ "INSERT INTO entities (id, ontology_id, name_cn, type, description, properties, confidence, version, created_at, updated_at) "
+ "VALUES ('e-1', :o, '概念', 'Concept', 'd', '{}'::json, 0.9, 'v1', now(), now())"
+ ), {"o": oid})
+ session.execute(text(
+ "INSERT INTO entity_instances (id, entity_id, ontology_id, row_identity, row_data, revision, created_at, updated_at) "
+ "VALUES (:i1, 'e-1', :o, 'inst-1', '{}'::json, 1, now(), now()), "
+ "(:i2, 'e-1', :o, 'inst-2', '{}'::json, 1, now(), now())"
+ ), {"o": oid, "i1": str(uuid.uuid4()), "i2": str(uuid.uuid4())})
+ inst_ids = session.execute(text(
+ "SELECT id FROM entity_instances WHERE ontology_id = :o ORDER BY row_identity"
+ ), {"o": oid}).scalars().all()
+ session.execute(text(
+ "INSERT INTO relations (id, ontology_id, source_entity, target_entity, type, properties, confidence, created_at) "
+ "VALUES ('r-1', :o, 'e-1', 'e-1', 'related_to', '{}'::json, 0.9, now())"
+ ), {"o": oid})
+ session.execute(text(
+ "INSERT INTO entity_instance_relations (id, ontology_id, source_instance_id, target_instance_id, relation_definition_id, properties, revision, created_at, updated_at) "
+ "VALUES (:id, :o, :s, :t, 'r-1', '{}'::json, 1, now(), now())"
+ ), {"id": str(uuid.uuid4()), "o": oid, "s": inst_ids[0], "t": inst_ids[1]})
+ session.commit()
+
+ # confirm the trigger fired on insert too (sanity: outbox has rows before delete)
+ outbox_before = session.execute(text(
+ "SELECT count(*) FROM agent_index_outbox WHERE ontology_id = :o"
+ ), {"o": oid}).scalar_one()
+ assert outbox_before > 0
+
+ # rows were inserted via direct SQL, bypassing create_ontology's creator grant
+ backfill_creator_grants(session)
+ delete_ontology(ontology_id=oid, db=session, current_user=session.get(User, u))
+
+ remaining = session.execute(text(
+ "SELECT count(*) FROM ontology_projects WHERE id = :o"
+ ), {"o": oid}).scalar_one()
+ assert remaining == 0
diff --git a/backend/tests/test_python_version.py b/backend/tests/test_python_version.py
new file mode 100644
index 00000000..a0fdd391
--- /dev/null
+++ b/backend/tests/test_python_version.py
@@ -0,0 +1,18 @@
+import pytest
+
+from scripts import check_python_version
+
+
+@pytest.mark.parametrize("version", [(3, 11, 0), (3, 12, 0)])
+def test_supported_python_versions_are_accepted(monkeypatch, version):
+ monkeypatch.setattr(check_python_version.sys, "version_info", version)
+
+ assert check_python_version.require_supported_python() is True
+
+
+@pytest.mark.parametrize("version", [(3, 10, 0), (3, 13, 0)])
+def test_python_versions_outside_supported_range_are_rejected(monkeypatch, version):
+ monkeypatch.setattr(check_python_version.sys, "version_info", version)
+
+ with pytest.raises(SystemExit, match="UNSUPPORTED_PYTHON_VERSION"):
+ check_python_version.require_supported_python()
diff --git a/backend/tests/v2/connection/test_connection_resources.py b/backend/tests/v2/connection/test_connection_resources.py
new file mode 100644
index 00000000..5d7ce171
--- /dev/null
+++ b/backend/tests/v2/connection/test_connection_resources.py
@@ -0,0 +1,138 @@
+"""v2 saved database connection resource and sample API coverage."""
+import json
+from unittest.mock import MagicMock, patch
+
+from app.models.v2.connection import Connection
+from app.models.user import User
+from app.routers.v2 import connections as connections_router
+from app.services.auth_service import create_access_token, hash_password
+
+
+def _override_connections_db(db):
+ def override():
+ yield db
+
+ return override
+
+
+def test_saved_connection_keeps_credentials_encrypted_and_lists_resources(
+ client, db, auth_headers, admin_user, monkeypatch,
+):
+ app = client.app
+ previous = app.dependency_overrides.get(connections_router.get_db)
+ app.dependency_overrides[connections_router.get_db] = _override_connections_db(db)
+ try:
+ created = client.post(
+ "/api/v2/connections",
+ headers=auth_headers,
+ json={
+ "name": "runtime mysql",
+ "kind": "mysql",
+ "config": {
+ "host": "127.0.0.1",
+ "port": "53306",
+ "database": "runtime",
+ "user": "runtime",
+ "password": "secret",
+ },
+ },
+ )
+ assert created.status_code == 201
+ connection = db.get(Connection, created.json()["id"])
+ assert connection is not None
+ assert "secret" not in json.dumps(connection.config)
+ assert "_encrypted" in connection.config
+ assert connection.created_by == admin_user.id
+
+ fake = MagicMock()
+ fake.list_resources.return_value = ["codex_pipeline_probe"]
+ monkeypatch.setattr(connections_router, "get_connector", lambda kind, config: fake)
+ listed = client.get(
+ f"/api/v2/connections/{connection.id}/resources",
+ headers=auth_headers,
+ )
+
+ assert listed.status_code == 200
+ assert listed.json()["resources"] == ["codex_pipeline_probe"]
+ fake.list_resources.assert_called_once_with()
+ finally:
+ if previous is None:
+ app.dependency_overrides.pop(connections_router.get_db, None)
+ else:
+ app.dependency_overrides[connections_router.get_db] = previous
+
+
+def test_saved_connection_returns_sample_rows_for_a_table(client, db, auth_headers, monkeypatch):
+ app = client.app
+ previous = app.dependency_overrides.get(connections_router.get_db)
+ app.dependency_overrides[connections_router.get_db] = _override_connections_db(db)
+ try:
+ connection = Connection(
+ id="resource-sample-connection",
+ name="runtime postgres",
+ kind="postgres",
+ config={"_encrypted": "ignored-by-mock"},
+ status="inactive",
+ )
+ db.add(connection)
+ db.commit()
+
+ fake = MagicMock()
+ fake.pull_sample.return_value = [{"id": 1, "customer": "Alice"}]
+ monkeypatch.setattr(connections_router, "_load_connection_connector", lambda *args, **kwargs: fake)
+ response = client.get(
+ f"/api/v2/connections/{connection.id}/resources/codex_pipeline_probe/sample?limit=1",
+ headers=auth_headers,
+ )
+
+ assert response.status_code == 200
+ assert response.json()["rows"] == [{"id": 1, "customer": "Alice"}]
+ fake.pull_sample.assert_called_once_with("codex_pipeline_probe", limit=1)
+ finally:
+ if previous is None:
+ app.dependency_overrides.pop(connections_router.get_db, None)
+ else:
+ app.dependency_overrides[connections_router.get_db] = previous
+
+
+def test_viewer_cannot_list_or_sample_saved_connection_resources(client, db):
+ app = client.app
+ previous = app.dependency_overrides.get(connections_router.get_db)
+ app.dependency_overrides[connections_router.get_db] = _override_connections_db(db)
+ try:
+ db.add(Connection(
+ id="viewer-resource-connection",
+ name="runtime postgres",
+ kind="postgres",
+ config={"_encrypted": "ignored-by-authz-test"},
+ status="active",
+ ))
+ viewer = User(
+ id="resource-viewer",
+ username="resource-viewer",
+ email="resource-viewer@test.com",
+ password_hash=hash_password("viewer123"),
+ role="viewer",
+ )
+ db.add(viewer)
+ db.commit()
+ headers = {
+ "Authorization": f"Bearer {create_access_token({'sub': viewer.id, 'role': viewer.role})}",
+ }
+
+ listed = client.get(
+ "/api/v2/connections/viewer-resource-connection/resources",
+ headers=headers,
+ )
+ sampled = client.get(
+ "/api/v2/connections/viewer-resource-connection/resources/orders/sample",
+ headers=headers,
+ )
+
+ assert listed.status_code == 403
+ assert sampled.status_code == 403
+ finally:
+ if previous is None:
+ app.dependency_overrides.pop(connections_router.get_db, None)
+ else:
+ app.dependency_overrides[connections_router.get_db] = previous
diff --git a/backend/tests/v2/connection/test_scheduler.py b/backend/tests/v2/connection/test_scheduler.py
index 1c086e89..287753e3 100644
--- a/backend/tests/v2/connection/test_scheduler.py
+++ b/backend/tests/v2/connection/test_scheduler.py
@@ -47,9 +47,8 @@ def test_schedule_connection_without_db_only_validates_never_reports_active():
def test_schedule_pipeline_without_db_only_validates_never_reports_active():
svc = CronService()
- result = svc.schedule_pipeline_run("pl-1", "*/30 * * * *")
- assert result["status"] != "scheduled"
- assert result["pipeline_id"] == "pl-1"
+ with pytest.raises(ValueError, match="PIPELINE_SCHEDULE_UNSUPPORTED"):
+ svc.schedule_pipeline_run("pl-1", "*/30 * * * *")
def test_schedule_connection_with_db_persists_and_returns_scheduled(db):
@@ -69,10 +68,13 @@ def test_schedule_connection_with_db_persists_and_returns_scheduled(db):
def test_schedule_pipeline_with_db_persists_and_returns_scheduled(db):
svc = CronService()
- result = svc.schedule_pipeline_run("pl-1", "*/30 * * * *", db=db)
- assert result["status"] == "scheduled"
- assert result["pipeline_id"] == "pl-1"
- assert result["next_due_at"] is not None
+ with pytest.raises(ValueError, match="PIPELINE_SCHEDULE_UNSUPPORTED"):
+ svc.schedule_pipeline_run("pl-1", "*/30 * * * *", db=db)
+
+ from app.models.v2.refresh import RefreshSchedule
+ assert db.query(RefreshSchedule).filter(
+ RefreshSchedule.target_type == "pipeline", RefreshSchedule.target_id == "pl-1",
+ ).count() == 0
def test_describe_cron_every_minute():
@@ -137,11 +139,11 @@ def scheduler_connection(db):
db.add(Connection(
id="scheduler-conn-001", name="scheduler-source", kind="rest", status="active",
- config={"cursor_contract": "watermark_primary_key"},
+ config={"cursor_contract": "watermark_primary_key", "resource": "orders"},
refresh_policy="micro_batch", cursor_contract="watermark_primary_key",
))
db.add(RefreshSourceState(
- id="scheduler-state-001", source_id="scheduler-conn-001", resource="__scheduled__",
+ id="scheduler-state-001", source_id="scheduler-conn-001", resource="orders",
cursor_contract="watermark_primary_key", config_version=1,
cursor_json={
"watermark": NOW.isoformat(), "primary_key": "1", "opaque_value": None,
diff --git a/backend/tests/v2/connection/test_sql_connector.py b/backend/tests/v2/connection/test_sql_connector.py
index 46acd331..79d34553 100644
--- a/backend/tests/v2/connection/test_sql_connector.py
+++ b/backend/tests/v2/connection/test_sql_connector.py
@@ -37,6 +37,49 @@ def test_registry_mysql_returns_sql_connector():
assert CONNECTOR_REGISTRY["postgres"] is SQLConnector
+def test_saved_sql_connection_fields_are_compiled_to_a_safe_sqlalchemy_url():
+ from app.services.connection.config import build_connector_config
+
+ mysql = build_connector_config(
+ "mysql",
+ {
+ "host": "127.0.0.1",
+ "port": "3306",
+ "database": "runtime",
+ "user": "runtime",
+ "password": "p@ss:word/with#chars",
+ },
+ )
+ postgres = build_connector_config(
+ "postgres",
+ {
+ "host": "127.0.0.1",
+ "port": 5432,
+ "database": "runtime",
+ "user": "runtime",
+ "password": "secret",
+ },
+ )
+
+ assert mysql["connection_string"] == (
+ "mysql+pymysql://runtime:p%40ss%3Aword%2Fwith%23chars@127.0.0.1:3306/runtime"
+ )
+ assert postgres["connection_string"] == "postgresql+psycopg2://runtime:secret@127.0.0.1:5432/runtime"
+
+
+def test_sql_connector_keeps_actionable_failure_detail_without_exposing_password():
+ from app.services.connection.sql_connector import SQLConnector
+
+ connector = SQLConnector({
+ "connection_string": "postgresql+psycopg2://user:secret@localhost:5432/runtime",
+ })
+ connector._get_engine = MagicMock(side_effect=RuntimeError("connection refused"))
+
+ assert connector.test_connection() is False
+ assert connector.last_error == "connection refused"
+ assert "secret" not in connector.last_error
+
+
def test_get_connector_all_supported_kinds():
from app.services.connection.registry import get_connector
for kind in ["file", "mysql", "postgres"]:
diff --git a/backend/tests/v2/graph/test_analytics.py b/backend/tests/v2/graph/test_analytics.py
index 34e83d0b..dc79b532 100644
--- a/backend/tests/v2/graph/test_analytics.py
+++ b/backend/tests/v2/graph/test_analytics.py
@@ -63,3 +63,39 @@ def test_node_degree_with_result():
result = svc.node_degree("ont-1", "node-1")
assert result["in_degree"] == 3
assert result["out_degree"] == 7
+
+
+def test_graph_traversals_scope_every_returned_node_to_ontology():
+ svc, mock_neo4j = make_analytics(available=True)
+ mock_neo4j.run_cypher.return_value = []
+
+ svc.get_neighbors("ont-1", "node-1")
+ neighbors_query = mock_neo4j.run_cypher.call_args.args[0]
+ assert "all(node IN nodes(path) WHERE node.ontology_id = $ontology_id)" in neighbors_query
+
+ svc.shortest_path("ont-1", "a", "b")
+ path_query = mock_neo4j.run_cypher.call_args.args[0]
+ assert "all(node IN nodes(p) WHERE node.ontology_id = $ontology_id)" in path_query
+
+ svc.node_degree("ont-1", "node-1")
+ degree_query = mock_neo4j.run_cypher.call_args.args[0]
+ assert "out_neighbor.ontology_id = $ontology_id" in degree_query
+ assert "in_neighbor.ontology_id = $ontology_id" in degree_query
+
+ svc.top_connected_nodes("ont-1")
+ top_query = mock_neo4j.run_cypher.call_args.args[0]
+ assert "neighbor.ontology_id = $ontology_id" in top_query
+
+
+def test_graph_neighbors_route_scopes_every_path_node(monkeypatch):
+ from app.routers.v2 import graph
+
+ fake_neo4j = MagicMock()
+ fake_neo4j.available = True
+ fake_neo4j.run_cypher.return_value = []
+ monkeypatch.setattr(graph, "get_neo4j", lambda: fake_neo4j)
+
+ graph.get_neighbors("ont-1", "node-1")
+
+ query = fake_neo4j.run_cypher.call_args.args[0]
+ assert "all(node IN nodes(path) WHERE node.ontology_id = $ontology_id)" in query
diff --git a/backend/tests/v2/incremental/test_refresh_schedule.py b/backend/tests/v2/incremental/test_refresh_schedule.py
index 9781dd4b..7933ecc3 100644
--- a/backend/tests/v2/incremental/test_refresh_schedule.py
+++ b/backend/tests/v2/incremental/test_refresh_schedule.py
@@ -24,9 +24,9 @@
import pytest
from sqlalchemy import func, select
+from app.models.v2.connection import Connection
from app.models.v2.refresh import RefreshRun, RefreshSchedule
from app.services.v2.scheduler.schedule_service import (
- DEFAULT_RESOURCE,
ScheduleRequest,
dispatch_due_schedules,
upsert_refresh_schedule,
@@ -51,6 +51,12 @@ def upsert_fixture_schedule(db, *, cron_expr, timezone, business_calendar, sla_s
def create_fixture_schedule(db, *, target_id, max_pending_runs=5, cron_expr="* * * * *", retry_policy=None) -> RefreshSchedule:
"""A schedule already due at FIXED_NOW (every-minute cron, upserted one
minute before FIXED_NOW so next_due_at lands exactly on it)."""
+ if db.get(Connection, target_id) is None:
+ db.add(Connection(
+ id=target_id, name=target_id, kind="rest", config={"resource": "orders"},
+ status="active", refresh_policy="batch", cursor_contract="watermark_primary_key",
+ ))
+ db.commit()
request = ScheduleRequest(
target_type="connection", target_id=target_id, cron_expr=cron_expr, timezone="UTC",
business_calendar=[], sla_seconds=0, retry_policy=retry_policy, backfill_window_seconds=0,
@@ -59,7 +65,7 @@ def create_fixture_schedule(db, *, target_id, max_pending_runs=5, cron_expr="* *
return upsert_refresh_schedule(db, request, now=FIXED_NOW - timedelta(minutes=1))
-def create_queued_fixture_run(db, *, source_id, resource=DEFAULT_RESOURCE) -> RefreshRun:
+def create_queued_fixture_run(db, *, source_id, resource="orders") -> RefreshRun:
"""A pre-existing active run occupying the source's admission slot,
independent of any schedule dispatch."""
run = RefreshRun(
@@ -123,6 +129,12 @@ def test_scheduled_run_persists_destination_queue_for_operability(db):
def test_manual_refresh_run_persists_destination_queue_for_operability(db):
from app.tasks.v2.refresh_tasks import create_manual_connection_run
+ db.add(Connection(
+ id="source-manual-queue-attribution", name="manual source", kind="rest",
+ config={"resource": "orders"}, status="active", refresh_policy="batch",
+ cursor_contract="watermark_primary_key",
+ ))
+ db.commit()
run = create_manual_connection_run(db, "source-manual-queue-attribution")
assert run.dispatch_queue == "refresh.poll"
@@ -287,7 +299,7 @@ def test_expired_dispatch_claim_is_recovered_once_without_duplicate(db):
schedule = create_fixture_schedule(db, target_id="source-expired-claim")
occurrence_key = f"schedule:{schedule.id}:{schedule.next_due_at.isoformat()}"
retained = RefreshRun(
- id=str(uuid.uuid4()), source_id="source-expired-claim", resource=DEFAULT_RESOURCE,
+ id=str(uuid.uuid4()), source_id="source-expired-claim", resource="orders",
policy="batch", trigger="scheduled", config_version=1,
cursor_contract="watermark_primary_key", status="queued", dispatch_state="pending",
dispatch_queue="refresh.poll", dispatch_claim_owner="abandoned-beat",
diff --git a/backend/tests/v2/incremental/test_resource_resolution.py b/backend/tests/v2/incremental/test_resource_resolution.py
new file mode 100644
index 00000000..a6c223f8
--- /dev/null
+++ b/backend/tests/v2/incremental/test_resource_resolution.py
@@ -0,0 +1,195 @@
+"""Deterministic server-owned resource selection for refresh entry points."""
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+from app.models.v2.connection import Connection
+from app.models.v2.dataset import Dataset
+from app.models.v2.refresh import RefreshRun, RefreshSchedule, RefreshSourceState
+from app.schemas.refresh import RefreshError, RefreshPolicy
+from app.services.v2.incremental.operations import resolve_refresh_resource
+from app.services.v2.scheduler.schedule_service import dispatch_due_schedules, upsert_refresh_schedule, ScheduleRequest
+from app.tasks.v2.refresh_tasks import create_manual_connection_run
+
+
+NOW = datetime(2026, 9, 24, tzinfo=timezone.utc)
+
+
+def _connection(db, connection_id="resource-source", config=None):
+ connection = Connection(
+ id=connection_id,
+ name=connection_id,
+ kind="rest",
+ config=config or {},
+ status="active",
+ refresh_policy=RefreshPolicy.BATCH.value,
+ cursor_contract="watermark_primary_key",
+ )
+ db.add(connection)
+ db.commit()
+ return connection
+
+
+def test_resource_config_wins_over_ambiguous_states(db):
+ _connection(db, config={"resource": "orders"})
+ db.add_all([
+ RefreshSourceState(
+ id="state-users", source_id="resource-source", resource="users",
+ cursor_contract="watermark_primary_key", config_version=1,
+ fencing_token=0,
+ ),
+ RefreshSourceState(
+ id="state-events", source_id="resource-source", resource="events",
+ cursor_contract="watermark_primary_key", config_version=1,
+ fencing_token=0,
+ ),
+ ])
+ db.commit()
+
+ assert resolve_refresh_resource(db, source_id="resource-source") == "orders"
+
+
+def test_resource_resolution_ignores_legacy_sentinel_and_uses_one_real_state(db):
+ _connection(db)
+ db.add(RefreshSourceState(
+ id="state-sentinel", source_id="resource-source", resource="__scheduled__",
+ cursor_contract="watermark_primary_key", config_version=1, fencing_token=0,
+ ))
+ db.commit()
+
+ with pytest.raises(RefreshError) as exc_info:
+ resolve_refresh_resource(db, source_id="resource-source")
+ assert exc_info.value.reason_code == "REFRESH_RESOURCE_REQUIRED"
+
+ db.add(RefreshSourceState(
+ id="state-orders", source_id="resource-source", resource="orders",
+ cursor_contract="watermark_primary_key", config_version=1, fencing_token=0,
+ ))
+ db.commit()
+ assert resolve_refresh_resource(db, source_id="resource-source") == "orders"
+
+
+def test_resource_resolution_uses_one_dataset_when_state_is_absent(db):
+ _connection(db)
+ db.add(Dataset(
+ id="dataset-orders", name="orders", source_connection_id="resource-source",
+ kind="structured",
+ ))
+ db.commit()
+
+ assert resolve_refresh_resource(db, source_id="resource-source") == "orders"
+
+
+def test_resource_resolution_rejects_ambiguous_state_candidates(db):
+ _connection(db)
+ db.add_all([
+ RefreshSourceState(
+ id="state-orders", source_id="resource-source", resource="orders",
+ cursor_contract="watermark_primary_key", config_version=1, fencing_token=0,
+ ),
+ RefreshSourceState(
+ id="state-users", source_id="resource-source", resource="users",
+ cursor_contract="watermark_primary_key", config_version=1, fencing_token=0,
+ ),
+ ])
+ db.commit()
+
+ with pytest.raises(RefreshError) as exc_info:
+ resolve_refresh_resource(db, source_id="resource-source")
+ assert exc_info.value.reason_code == "REFRESH_RESOURCE_AMBIGUOUS"
+
+
+def test_manual_compatibility_run_uses_concrete_resolved_resource(db):
+ _connection(db, config={"resource": "orders"})
+
+ run = create_manual_connection_run(db, "resource-source")
+
+ assert run.resource == "orders"
+ assert db.query(RefreshRun).filter(
+ RefreshRun.source_id == "resource-source", RefreshRun.resource == "__scheduled__",
+ ).count() == 0
+
+
+def test_due_source_schedule_uses_concrete_resolved_resource(db):
+ _connection(db, config={"resource": "orders"})
+ schedule = upsert_refresh_schedule(
+ db,
+ ScheduleRequest(
+ target_type="connection", target_id="resource-source", cron_expr="* * * * *",
+ timezone="UTC", business_calendar=[], sla_seconds=0, retry_policy=None,
+ backfill_window_seconds=0, max_pending_runs=1, enabled=True,
+ ),
+ now=NOW - timedelta(minutes=1),
+ )
+
+ sent = []
+ dispatched = dispatch_due_schedules(
+ db, now=NOW, lease_owner="beat", send_refresh=lambda message: sent.append(message) or message.run_id,
+ )
+
+ run = db.query(RefreshRun).filter(RefreshRun.idempotency_key.like(f"schedule:{schedule.id}:%")).one()
+ assert dispatched == [run.id]
+ assert run.resource == "orders"
+ assert run.resource != "__scheduled__"
+ assert sent[0].task_name == "refresh.connection"
+
+
+def test_pipeline_target_schedule_is_rejected_before_persistence(db):
+ from app.services.v2.scheduler.cron_service import CronService
+
+ with pytest.raises(ValueError, match="PIPELINE_SCHEDULE_UNSUPPORTED"):
+ CronService().schedule_pipeline_run("pipeline-1", "*/30 * * * *", db=db)
+
+ assert db.query(RefreshSchedule).filter(
+ RefreshSchedule.target_type == "pipeline", RefreshSchedule.target_id == "pipeline-1",
+ ).count() == 0
+
+
+def test_existing_pipeline_schedule_is_disabled_without_poll_run(db):
+ schedule = RefreshSchedule(
+ id="pipeline-schedule-1", target_type="pipeline", target_id="pipeline-1",
+ cron_expression="* * * * *", timezone="UTC", excluded_dates=[],
+ max_pending_runs=1, enabled=True, next_due_at=NOW,
+ )
+ db.add(schedule)
+ db.commit()
+
+ assert dispatch_due_schedules(
+ db, now=NOW, lease_owner="beat", send_refresh=lambda message: message.run_id,
+ ) == []
+ db.refresh(schedule)
+ assert schedule.enabled is False
+ assert db.query(RefreshRun).filter(RefreshRun.source_id == "pipeline-1").count() == 0
+
+
+def test_bad_resource_schedule_does_not_block_good_schedule(db):
+ _connection(db, connection_id="good-resource-source", config={"resource": "orders"})
+ _connection(db, connection_id="bad-resource-source")
+ good = upsert_refresh_schedule(
+ db,
+ ScheduleRequest(
+ target_type="connection", target_id="good-resource-source", cron_expr="* * * * *",
+ timezone="UTC", business_calendar=[], sla_seconds=0, retry_policy=None,
+ backfill_window_seconds=0, max_pending_runs=1, enabled=True,
+ ),
+ now=NOW - timedelta(minutes=1),
+ )
+ bad = upsert_refresh_schedule(
+ db,
+ ScheduleRequest(
+ target_type="connection", target_id="bad-resource-source", cron_expr="* * * * *",
+ timezone="UTC", business_calendar=[], sla_seconds=0, retry_policy=None,
+ backfill_window_seconds=0, max_pending_runs=1, enabled=True,
+ ),
+ now=NOW - timedelta(minutes=1),
+ )
+
+ dispatched = dispatch_due_schedules(
+ db, now=NOW, lease_owner="beat", send_refresh=lambda message: message.run_id,
+ )
+
+ assert len(dispatched) == 1
+ run = db.query(RefreshRun).filter(RefreshRun.idempotency_key.like(f"schedule:{good.id}:%")).one()
+ assert dispatched == [run.id]
+ db.refresh(bad)
+ assert bad.next_due_at.replace(tzinfo=timezone.utc) > NOW
diff --git a/backend/tests/v2/pipeline/test_database_source.py b/backend/tests/v2/pipeline/test_database_source.py
new file mode 100644
index 00000000..7d5925ad
--- /dev/null
+++ b/backend/tests/v2/pipeline/test_database_source.py
@@ -0,0 +1,126 @@
+"""Database connector source -> snapshot -> Route A pipeline coverage."""
+from unittest.mock import MagicMock
+
+from sqlalchemy.orm import sessionmaker
+
+from app.models.v2.connection import Connection
+from app.models.v2.pipeline import Pipeline, PipelineRun, PipelineRunInput
+from app.services import encryption_service
+from app.tasks.v2 import pipeline_run as pipeline_module
+from app.routers.v2.pipelines import _sanitize_definition
+
+
+def test_pipeline_definition_strips_database_credentials_before_persisting():
+ definition = {
+ "nodes": [{
+ "type": "connector",
+ "config": {
+ "source_type": "postgresql",
+ "connection_id": "db-source-001",
+ "resource": "codex_pipeline_probe",
+ "config_values": {
+ "host": "127.0.0.1",
+ "password": "do-not-store",
+ },
+ "password": "do-not-store-either",
+ "credentials": {
+ "username": "runtime",
+ "password": "nested-do-not-store",
+ },
+ },
+ }],
+ "edges": [],
+ }
+
+ sanitized = _sanitize_definition(definition)
+
+ assert sanitized is not definition
+ config = sanitized["nodes"][0]["config"]
+ assert config["connection_id"] == "db-source-001"
+ assert "password" not in config
+ assert "config_values" not in config
+ assert "credentials" not in config
+ assert definition["nodes"][0]["config"]["password"] == "do-not-store-either"
+
+
+def test_pipeline_run_snapshots_selected_database_table_and_records_lineage(db, monkeypatch):
+ class MemoryStorage:
+ def __init__(self):
+ self.objects = {}
+
+ def put_bytes(self, bucket, key, data, content_type="application/octet-stream"):
+ uri = f"s3://{bucket}/{key}"
+ self.objects[uri] = data
+ return uri
+
+ def get_object(self, uri):
+ return self.objects[uri]
+
+ storage = MemoryStorage()
+ monkeypatch.setattr("app.database.SessionLocal", sessionmaker(bind=db.get_bind()))
+ monkeypatch.setattr("app.services.v2.dataset_service.get_storage_service", lambda: storage)
+ connector = MagicMock()
+ connector.pull_full.return_value = [
+ {"id": row_id, "customer": f"Customer {row_id}", "amount": row_id}
+ for row_id in range(10001)
+ ]
+ monkeypatch.setattr(pipeline_module, "get_connector", lambda kind, config: connector)
+ monkeypatch.setattr(
+ pipeline_module,
+ "_execute_route",
+ lambda route, context, data: (data, context),
+ )
+
+ connection = Connection(
+ id="db-source-001",
+ name="runtime postgres",
+ kind="postgres",
+ config={"_encrypted": encryption_service.encrypt(
+ '{"host":"127.0.0.1","port":"55432","database":"runtime",'
+ '"user":"runtime","password":"secret"}'
+ )},
+ status="active",
+ )
+ pipeline = Pipeline(
+ id="db-pipeline-001",
+ name="database pipeline",
+ route="A",
+ spec={},
+ definition={
+ "nodes": [{
+ "id": "source",
+ "type": "connector",
+ "config": {
+ "source_type": "postgresql",
+ "connection_id": connection.id,
+ "resource": "codex_pipeline_probe",
+ },
+ }],
+ "edges": [],
+ },
+ status="draft",
+ )
+ run = PipelineRun(id="db-run-001", pipeline_id=pipeline.id, status="pending")
+ db.add_all([connection, pipeline, run])
+ db.commit()
+
+ pipeline_module.pipeline_run_task.run(pipeline.id, run.id)
+ db.expire_all()
+
+ persisted = db.get(PipelineRun, run.id)
+ assert persisted.status == "success"
+ assert persisted.dataset_version_id is not None
+ assert persisted.stats["rows_in"] == 10001
+ assert db.query(PipelineRunInput).filter(
+ PipelineRunInput.pipeline_run_id == run.id,
+ ).count() >= 2
+ connector.pull_full.assert_called_once_with("codex_pipeline_probe")
+ from app.models.v2.dataset import Dataset
+ from app.services.v2.incremental.polling import _resolve_pipeline
+
+ source_dataset = db.query(Dataset).filter(
+ Dataset.source_connection_id == connection.id,
+ Dataset.name == "codex_pipeline_probe",
+ ).one()
+ assert db.get(Pipeline, pipeline.id).source_dataset_id == source_dataset.id
+ assert _resolve_pipeline(db, dataset=source_dataset).id == pipeline.id
diff --git a/backend/tests/v2/pipeline/test_pinned_inputs.py b/backend/tests/v2/pipeline/test_pinned_inputs.py
new file mode 100644
index 00000000..b24d9c47
--- /dev/null
+++ b/backend/tests/v2/pipeline/test_pinned_inputs.py
@@ -0,0 +1,104 @@
+"""Pipeline input pinning and pipeline-level status ordering regressions."""
+from datetime import datetime, timezone
+from sqlalchemy.orm import sessionmaker
+
+from app.models.v2.dataset import Dataset, DatasetVersion
+from app.models.v2.pipeline import Pipeline, PipelineRun
+from app.tasks.v2 import pipeline_run as pipeline_module
+
+
+def _file_pipeline(db, *, pipeline_id="pinned-pipeline"):
+ first = Dataset(id=f"{pipeline_id}-dataset-a", name="a", kind="structured")
+ second = Dataset(id=f"{pipeline_id}-dataset-b", name="b", kind="structured")
+ first_version = DatasetVersion(
+ id=f"{pipeline_id}-version-a", dataset_id=first.id, version_no=1, rowcount=1,
+ )
+ second_version = DatasetVersion(
+ id=f"{pipeline_id}-version-b", dataset_id=second.id, version_no=1, rowcount=1,
+ )
+ pipeline = Pipeline(
+ id=pipeline_id, name=pipeline_id, spec={}, status="draft",
+ definition={
+ "nodes": [{
+ "id": "source-a", "type": "connector",
+ "config": {"source_type": "file", "files": [{"dataset_id": first.id, "name": "a.csv"}]},
+ }, {
+ "id": "source-b", "type": "connector",
+ "config": {"source_type": "file", "files": [{"dataset_id": second.id, "name": "b.csv"}]},
+ }],
+ "edges": [],
+ },
+ )
+ db.add_all([first, second, first_version, second_version, pipeline])
+ db.commit()
+ return pipeline, first_version, second_version
+
+
+def test_incomplete_pinned_inputs_fail_before_latest_lookup(db, monkeypatch):
+ pipeline, first_version, _ = _file_pipeline(db)
+ run = PipelineRun(id="pinned-run-incomplete", pipeline_id=pipeline.id, status="pending")
+ db.add(run)
+ db.commit()
+
+ def fail_if_latest_lookup(*args, **kwargs):
+ raise AssertionError("latest version lookup must not run for incomplete pins")
+
+ monkeypatch.setattr(
+ "app.services.v2.incremental.polling.select_pinned_dataset_version",
+ fail_if_latest_lookup,
+ )
+ monkeypatch.setattr("app.database.SessionLocal", sessionmaker(bind=db.get_bind()))
+ pipeline_module.pipeline_run_task.run(pipeline.id, run.id, [first_version.id])
+ db.expire_all()
+
+ persisted = db.get(PipelineRun, run.id)
+ assert persisted.status == "failed"
+ assert persisted.error_log.startswith("PINNED_INPUTS_INCOMPLETE")
+
+
+def test_duplicate_pins_for_one_source_are_rejected(db, monkeypatch):
+ pipeline, first_version, _ = _file_pipeline(db, pipeline_id="pinned-pipeline-duplicate")
+ replacement = DatasetVersion(
+ id="pinned-pipeline-duplicate-version-a-2",
+ dataset_id=first_version.dataset_id,
+ version_no=2,
+ rowcount=1,
+ )
+ db.add(replacement)
+ run = PipelineRun(id="pinned-run-duplicate", pipeline_id=pipeline.id, status="pending")
+ db.add(run)
+ db.commit()
+
+ monkeypatch.setattr("app.database.SessionLocal", sessionmaker(bind=db.get_bind()))
+ pipeline_module.pipeline_run_task.run(
+ pipeline.id, run.id, [first_version.id, replacement.id],
+ )
+ db.expire_all()
+
+ persisted = db.get(PipelineRun, run.id)
+ assert persisted.status == "failed"
+ assert persisted.error_log.startswith("PINNED_INPUTS_INCOMPLETE")
+
+
+def test_pipeline_status_summary_only_follows_latest_run(db):
+ from app.tasks.v2.pipeline_run import _set_pipeline_status_if_latest
+
+ pipeline = Pipeline(id="status-pipeline", name="status-pipeline", spec={}, status="draft")
+ older = PipelineRun(
+ id="status-run-a", pipeline_id=pipeline.id, status="failed",
+ finished_at=datetime(2026, 9, 24, 0, 0, tzinfo=timezone.utc),
+ error_log="older failure",
+ created_at=datetime(2026, 9, 24, 0, 0, tzinfo=timezone.utc),
+ )
+ newer = PipelineRun(
+ id="status-run-b", pipeline_id=pipeline.id, status="running",
+ created_at=datetime(2026, 9, 24, 0, 1, tzinfo=timezone.utc),
+ )
+ db.add_all([pipeline, older, newer])
+ db.commit()
+
+ assert _set_pipeline_status_if_latest(db, pipeline, older, "published") is False
+ assert db.get(Pipeline, pipeline.id).status == "draft"
+
+ assert _set_pipeline_status_if_latest(db, pipeline, newer, "published") is True
+ assert db.get(Pipeline, pipeline.id).status == "published"
diff --git a/backend/tests/v2/test_curated_deletion.py b/backend/tests/v2/test_curated_deletion.py
new file mode 100644
index 00000000..355fbe1a
--- /dev/null
+++ b/backend/tests/v2/test_curated_deletion.py
@@ -0,0 +1,106 @@
+"""Curated dataset deletion — MinIO object cleanup and legacy-table cleanup"""
+import uuid
+from datetime import datetime, timezone
+
+import pytest
+
+from app.models.v2.curated import CuratedDataset, CuratedReview
+from app.models.v2.dataset import Dataset, DatasetVersion, MediaItem
+
+
+@pytest.fixture
+def curated_client(client, db):
+ """v2 curated 路由用模块内 get_db, 需单独覆盖到测试库"""
+ from app.main import app
+ from app.routers.v2 import curated
+
+ def _override():
+ yield db
+
+ app.dependency_overrides[curated.get_db] = _override
+ yield client
+ app.dependency_overrides.pop(curated.get_db, None)
+
+
+def _login(client, username, password):
+ r = client.post("/api/v1/auth/login", json={"username": username, "password": password})
+ return {"Authorization": f"Bearer {r.json()['data']['access_token']}"}
+
+
+def _make_curated_with_media(db) -> tuple[str, list[str]]:
+ ds = Dataset(id=str(uuid.uuid4()), name=f"del-{uuid.uuid4().hex[:6]}", kind="curated")
+ db.add(ds)
+ db.flush()
+ ver = DatasetVersion(id=str(uuid.uuid4()), dataset_id=ds.id, version_no=1)
+ db.add(ver)
+ db.flush()
+ uris = [f"s3://curated/{uuid.uuid4().hex}.pdf", f"s3://curated/{uuid.uuid4().hex}.docx"]
+ media = MediaItem(
+ id=str(uuid.uuid4()), dataset_version_id=ver.id, media_type="pdf",
+ storage_uri=uris[0], ocr_result_uri=uris[1],
+ )
+ db.add(media)
+ db.commit()
+ return ds.id, uris
+
+
+def test_delete_curated_cleans_up_minio_objects(curated_client, db, admin_user, monkeypatch):
+ """Deleting a curated dataset must remove its MediaItem MinIO objects
+ (storage_uri and ocr_result_uri) — previously nothing in the delete path
+ ever called StorageService.delete_object, leaving them as permanent
+ orphaned storage."""
+ ds_id, expected_uris = _make_curated_with_media(db)
+ deleted = []
+
+ class FakeStorage:
+ def delete_object(self, uri):
+ deleted.append(uri)
+
+ monkeypatch.setattr("app.services.storage_service.get_storage_service", lambda: FakeStorage())
+
+ headers = _login(curated_client, "admin", "admin123")
+ r = curated_client.delete(f"/api/v2/curated/{ds_id}", headers=headers)
+ assert r.status_code == 204
+ assert sorted(deleted) == sorted(expected_uris)
+
+
+def test_delete_curated_removes_legacy_mirror_row(curated_client, db, admin_user, monkeypatch):
+ """Once a dataset has been through review, _ensure_curated_dataset_row
+ (and the older /review endpoint) lazily creates a v2_curated_datasets
+ row with the SAME id as the real v2_datasets row, purely as an FK bridge
+ for CuratedReview. Deleting the dataset must remove that mirror row too
+ — previously delete_curated only ever deleted the v2_datasets row and
+ the CuratedReview rows, leaving the CuratedDataset mirror as a permanent
+ orphan."""
+ monkeypatch.setattr("app.services.storage_service.get_storage_service", lambda: object())
+ ds_id, _ = _make_curated_with_media(db)
+ # simulate what a completed review leaves behind
+ db.add(CuratedDataset(id=ds_id, name="mirror", status="approved"))
+ db.add(CuratedReview(
+ id=str(uuid.uuid4()), curated_dataset_id=ds_id, status="approved",
+ decided_at=datetime.now(timezone.utc),
+ ))
+ db.commit()
+
+ headers = _login(curated_client, "admin", "admin123")
+ r = curated_client.delete(f"/api/v2/curated/{ds_id}", headers=headers)
+ assert r.status_code == 204
+
+ assert db.query(CuratedDataset).filter(CuratedDataset.id == ds_id).first() is None
+ assert db.query(CuratedReview).filter(CuratedReview.curated_dataset_id == ds_id).first() is None
+
+
+def test_delete_curated_survives_minio_failure(curated_client, db, admin_user, monkeypatch):
+ """MinIO cleanup is best-effort: a Postgres delete that already committed
+ must still return 204 even if object storage cleanup blows up."""
+ ds_id, _ = _make_curated_with_media(db)
+
+ class BrokenStorage:
+ def delete_object(self, uri):
+ raise ConnectionError("minio unreachable")
+
+ monkeypatch.setattr("app.services.storage_service.get_storage_service", lambda: BrokenStorage())
+
+ headers = _login(curated_client, "admin", "admin123")
+ r = curated_client.delete(f"/api/v2/curated/{ds_id}", headers=headers)
+ assert r.status_code == 204
diff --git a/backend/tests/v2/test_logic_actions_runtime.py b/backend/tests/v2/test_logic_actions_runtime.py
index 4ada5abc..98cc2369 100644
--- a/backend/tests/v2/test_logic_actions_runtime.py
+++ b/backend/tests/v2/test_logic_actions_runtime.py
@@ -2,6 +2,8 @@
from app.models.action import Action
from app.models.logic import LogicRule
from app.models.ontology import OntologyProject
+from app.models.user import User
+from app.services.ontology_access import creator_grant
from app.models.v2.logic import OntologyLogicRule
from app.models.v2.action import OntologyActionRun, OntologyActionType
from app.routers.actions import publish_actions
@@ -203,6 +205,8 @@ def test_action_runtime_rejects_missing_required_parameter(db):
def test_v1_logic_publish_syncs_v2_status(db):
ontology_id = "ont-runtime-5"
+ actor = User(id="runtime", username="runtime", email="runtime@example.test", password_hash="unused", role="editor")
+ db.add(actor)
db.add(OntologyProject(id=ontology_id, name=ontology_id, domain="test", created_by="runtime", security_domain_id="00000000-0000-0000-0000-000000000001"))
db.add(LogicRule(
id="logic-v1",
@@ -224,9 +228,10 @@ def test_v1_logic_publish_syncs_v2_status(db):
status="draft",
))
db.commit()
+ creator_grant(db, ontology_id, actor)
with pytest.raises(HTTPException) as exc:
- publish_logic_rules(ontology_id, db)
+ publish_logic_rules(ontology_id, db, actor)
assert "PUBLICATION_NOT_ENABLED" in str(exc.value.detail)
# direct tool publication is closed; nothing is published
assert db.query(LogicRule).filter_by(id="logic-v1").first().status == "draft"
@@ -235,6 +240,8 @@ def test_v1_logic_publish_syncs_v2_status(db):
def test_v1_action_publish_syncs_v2_status(db):
ontology_id = "ont-runtime-6"
+ actor = User(id="runtime", username="runtime", email="runtime@example.test", password_hash="unused", role="editor")
+ db.add(actor)
db.add(OntologyProject(id=ontology_id, name=ontology_id, domain="test", created_by="runtime", security_domain_id="00000000-0000-0000-0000-000000000001"))
db.add(Action(
id="action-v1",
@@ -254,9 +261,10 @@ def test_v1_action_publish_syncs_v2_status(db):
status="draft",
))
db.commit()
+ creator_grant(db, ontology_id, actor)
with pytest.raises(HTTPException) as exc:
- publish_actions(ontology_id, db)
+ publish_actions(ontology_id, db, actor)
assert "PUBLICATION_NOT_ENABLED" in str(exc.value.detail)
# direct tool publication is closed; nothing is published
assert db.query(Action).filter_by(id="action-v1").first().status == "draft"
diff --git a/backend/tests/v2/test_pipeline_connector_safety.py b/backend/tests/v2/test_pipeline_connector_safety.py
new file mode 100644
index 00000000..c5180ce3
--- /dev/null
+++ b/backend/tests/v2/test_pipeline_connector_safety.py
@@ -0,0 +1,195 @@
+"""Builder pipeline connector safety contracts."""
+
+import uuid
+
+import pytest
+
+from app.models.v2.pipeline import Pipeline
+
+
+@pytest.fixture
+def pipelines_client(client, db):
+ """Use the test session for the v2 pipeline router."""
+ from app.main import app
+ from app.routers.v2 import pipelines
+
+ def _override():
+ yield db
+
+ app.dependency_overrides[pipelines.get_db] = _override
+ yield client
+ app.dependency_overrides.pop(pipelines.get_db, None)
+
+
+def _headers(client, username="admin", password="admin123"):
+ response = client.post(
+ "/api/v1/auth/login",
+ json={"username": username, "password": password},
+ )
+ return {"Authorization": f"Bearer {response.json()['data']['access_token']}"}
+
+
+def _inline_definition(source_type: str) -> dict:
+ return {
+ "nodes": [{
+ "id": "source",
+ "type": "connector",
+ "config": {
+ "source_type": source_type,
+ "config_values": {
+ "uri": "mongodb://user:secret@example.invalid/orders"
+ if source_type == "mongodb"
+ else "https://example.invalid/orders",
+ "headers": '{"Authorization":"Bearer secret"}',
+ },
+ },
+ }],
+ "edges": [],
+ }
+
+
+@pytest.mark.parametrize("source_type", ["mongodb", "rest_api"])
+def test_create_rejects_unsupported_inline_connectors_before_persistence(
+ pipelines_client, db, admin_user, source_type,
+):
+ name = f"unsupported-{source_type}-{uuid.uuid4().hex}"
+
+ response = pipelines_client.post(
+ "/api/v2/pipelines",
+ json={"name": name, "definition": _inline_definition(source_type)},
+ headers=_headers(pipelines_client),
+ )
+
+ assert response.status_code == 422
+ assert response.json()["detail"] == "UNSUPPORTED_INLINE_CONNECTOR"
+ assert db.query(Pipeline).filter(Pipeline.name == name).count() == 0
+
+
+def test_update_rejects_unsupported_inline_connector_before_mutating_definition(
+ pipelines_client, db, admin_user,
+):
+ pipeline = Pipeline(
+ name=f"existing-{uuid.uuid4().hex}",
+ route="A",
+ spec={},
+ definition={"nodes": [], "edges": []},
+ )
+ db.add(pipeline)
+ db.commit()
+ pipeline_id = pipeline.id
+
+ response = pipelines_client.put(
+ f"/api/v2/pipelines/{pipeline_id}",
+ json={"definition": _inline_definition("rest_api")},
+ headers=_headers(pipelines_client),
+ )
+
+ assert response.status_code == 422
+ assert response.json()["detail"] == "UNSUPPORTED_INLINE_CONNECTOR"
+ assert db.get(Pipeline, pipeline_id).definition == {"nodes": [], "edges": []}
+
+
+def test_validate_marks_legacy_unsupported_connector_definition_invalid(
+ pipelines_client, db, admin_user,
+):
+ pipeline = Pipeline(
+ name=f"legacy-{uuid.uuid4().hex}",
+ route="A",
+ spec={},
+ definition=_inline_definition("mongodb"),
+ )
+ db.add(pipeline)
+ db.commit()
+
+ response = pipelines_client.post(
+ f"/api/v2/pipelines/{pipeline.id}/validate",
+ headers=_headers(pipelines_client),
+ )
+
+ assert response.status_code == 200
+ payload = response.json()
+ assert payload["valid"] is False
+ assert payload["errors"] == [{
+ "node_id": "source",
+ "severity": "error",
+ "code": "UNSUPPORTED_INLINE_CONNECTOR",
+ "message": "MongoDB/REST inline connector nodes are unsupported in Builder; use File Upload in Builder and manage external connections in Data Connections.",
+ }]
+
+
+def test_legacy_inline_connector_secrets_are_not_returned_by_pipeline_detail(
+ pipelines_client, db, admin_user,
+):
+ pipeline = Pipeline(
+ name=f"legacy-secret-{uuid.uuid4().hex}",
+ route="A",
+ spec={},
+ definition=_inline_definition("rest_api"),
+ )
+ db.add(pipeline)
+ db.commit()
+
+ response = pipelines_client.get(
+ f"/api/v2/pipelines/{pipeline.id}",
+ headers=_headers(pipelines_client),
+ )
+
+ assert response.status_code == 200
+ assert response.json()["definition"]["nodes"][0]["config"] == {"source_type": "rest_api"}
+ assert "Bearer secret" not in response.text
+
+
+def test_legacy_inline_connector_cannot_queue_a_run(pipelines_client, db, admin_user):
+ pipeline = Pipeline(
+ name=f"legacy-run-{uuid.uuid4().hex}",
+ route="A",
+ spec={},
+ definition=_inline_definition("mongodb"),
+ status="draft",
+ )
+ db.add(pipeline)
+ db.commit()
+
+ response = pipelines_client.post(
+ f"/api/v2/pipelines/{pipeline.id}/run",
+ headers=_headers(pipelines_client),
+ )
+
+ assert response.status_code == 422
+ assert response.json()["detail"] == "UNSUPPORTED_INLINE_CONNECTOR"
+ db.refresh(pipeline)
+ assert pipeline.status == "draft"
+
+
+def test_saved_sql_source_does_not_store_legacy_inline_credentials(
+ pipelines_client, db, admin_user,
+):
+ definition = {
+ "nodes": [{
+ "id": "sql-source", "type": "connector",
+ "config": {
+ "source_type": "postgresql",
+ "connection_id": "saved-connection",
+ "resource": "orders",
+ "config_values": {
+ "uri": "postgresql://user:secret@example.invalid/orders",
+ "headers": {"Authorization": "Bearer secret"},
+ },
+ },
+ }],
+ "edges": [],
+ }
+ response = pipelines_client.post(
+ "/api/v2/pipelines",
+ json={"name": f"safe-sql-{uuid.uuid4().hex}", "definition": definition},
+ headers=_headers(pipelines_client),
+ )
+
+ assert response.status_code == 201
+ stored = db.get(Pipeline, response.json()["id"]).definition
+ assert stored["nodes"][0]["config"] == {
+ "source_type": "postgresql",
+ "connection_id": "saved-connection",
+ "resource": "orders",
+ }
+ assert "secret" not in response.text
diff --git a/backend/tests/v2/test_upload_limits.py b/backend/tests/v2/test_upload_limits.py
index 9fab92fe..be5551ed 100644
--- a/backend/tests/v2/test_upload_limits.py
+++ b/backend/tests/v2/test_upload_limits.py
@@ -2,6 +2,7 @@
import io
import pytest
+from fastapi import HTTPException
from app.config import settings
@@ -45,6 +46,36 @@ def test_upload_rejects_oversized_file(datasets_client, admin_user, monkeypatch)
assert r.status_code == 413
+@pytest.mark.asyncio
+async def test_bounded_upload_read_stops_after_limit_plus_one_byte(monkeypatch):
+ from app.routers.v2.datasets import _read_upload_bounded
+
+ limit = 8
+ payload = b"x" * (limit + 1000)
+
+ class TrackedUpload:
+ def __init__(self, value):
+ self.value = value
+ self.offset = 0
+ self.read_sizes = []
+
+ async def read(self, size=-1):
+ self.read_sizes.append(size)
+ end = min(self.offset + size, len(self.value))
+ chunk = self.value[self.offset:end]
+ self.offset = end
+ return chunk
+
+ upload = TrackedUpload(payload)
+
+ with pytest.raises(HTTPException) as error:
+ await _read_upload_bounded(upload, limit, chunk_size=4)
+
+ assert error.value.status_code == 413
+ assert upload.offset == limit + 1
+ assert upload.read_sizes == [4, 4, 1]
+
+
def test_upload_accepts_normal_csv(datasets_client, admin_user):
r = datasets_client.post(
"/api/v2/datasets/upload",
diff --git a/docker-compose.v2.yml b/docker-compose.v2.yml
index a5f66c9d..85c422da 100644
--- a/docker-compose.v2.yml
+++ b/docker-compose.v2.yml
@@ -78,6 +78,9 @@ services:
build: ./backend
ports: ["8000:8000"]
env_file: .env
+ # In-network clients use the container port, not the host's 8001 mapping.
+ environment: &chroma-environment
+ CHROMA_PORT: "8000"
depends_on:
migration:
condition: service_completed_successfully
@@ -103,6 +106,7 @@ services:
env_file: .env
profiles: ["refresh"]
environment:
+ <<: *chroma-environment
REFRESH_WORKER_CONCURRENCY: ${REFRESH_WORKER_CONCURRENCY:-2}
REFRESH_WORKER_PREFETCH_MULTIPLIER: ${REFRESH_WORKER_PREFETCH_MULTIPLIER:-1}
REFRESH_SOFT_TIME_LIMIT_SECONDS: ${REFRESH_SOFT_TIME_LIMIT_SECONDS:-300}
@@ -128,6 +132,7 @@ services:
env_file: .env
profiles: ["refresh"]
environment:
+ <<: *chroma-environment
REFRESH_WORKER_CONCURRENCY: ${REFRESH_WORKER_CONCURRENCY:-2}
REFRESH_WORKER_PREFETCH_MULTIPLIER: ${REFRESH_WORKER_PREFETCH_MULTIPLIER:-1}
REFRESH_SOFT_TIME_LIMIT_SECONDS: ${REFRESH_SOFT_TIME_LIMIT_SECONDS:-300}
@@ -153,6 +158,7 @@ services:
env_file: .env
profiles: ["refresh"]
environment:
+ <<: *chroma-environment
REFRESH_WORKER_CONCURRENCY: ${REFRESH_WORKER_CONCURRENCY:-2}
REFRESH_WORKER_PREFETCH_MULTIPLIER: ${REFRESH_WORKER_PREFETCH_MULTIPLIER:-1}
REFRESH_SOFT_TIME_LIMIT_SECONDS: ${REFRESH_SOFT_TIME_LIMIT_SECONDS:-300}
@@ -178,6 +184,7 @@ services:
env_file: .env
profiles: ["refresh"]
environment:
+ <<: *chroma-environment
REFRESH_WORKER_CONCURRENCY: ${REFRESH_WORKER_CONCURRENCY:-2}
REFRESH_WORKER_PREFETCH_MULTIPLIER: ${REFRESH_WORKER_PREFETCH_MULTIPLIER:-1}
REFRESH_SOFT_TIME_LIMIT_SECONDS: ${REFRESH_SOFT_TIME_LIMIT_SECONDS:-300}
@@ -202,6 +209,7 @@ services:
build: ./backend
env_file: .env
profiles: ["artifact"]
+ environment: *chroma-environment
depends_on:
migration:
condition: service_completed_successfully
@@ -220,6 +228,7 @@ services:
build: ./backend
env_file: .env
profiles: ["agent"]
+ environment: *chroma-environment
depends_on:
migration:
condition: service_completed_successfully
@@ -238,6 +247,7 @@ services:
build: ./backend
env_file: .env
profiles: ["housekeeping"]
+ environment: *chroma-environment
depends_on:
migration:
condition: service_completed_successfully
@@ -255,6 +265,7 @@ services:
celery_beat:
build: ./backend
env_file: .env
+ environment: *chroma-environment
depends_on:
migration:
condition: service_completed_successfully
diff --git a/frontend/src/api/client.test.ts b/frontend/src/api/client.test.ts
new file mode 100644
index 00000000..2e870e94
--- /dev/null
+++ b/frontend/src/api/client.test.ts
@@ -0,0 +1,42 @@
+import { afterAll, afterEach, beforeAll, expect, it, vi } from 'vitest'
+import { http, HttpResponse } from 'msw'
+import { setupServer } from 'msw/node'
+import { apiClient, apiClientV2 } from './client'
+import { useAuthStore } from '@/stores/authStore'
+
+const server = setupServer()
+beforeAll(() => server.listen({ onUnhandledRequest: 'error' }))
+afterAll(() => server.close())
+afterEach(() => {
+ server.resetHandlers()
+ vi.restoreAllMocks()
+ useAuthStore.setState({ token: null, user: null })
+})
+
+it.each([['v1', apiClient], ['v2', apiClientV2]] as const)('%s permission denial preserves login without refreshing', async (version, client) => {
+ useAuthStore.setState({ token: 'valid-session' })
+ const fetch = vi.spyOn(globalThis, 'fetch')
+ server.use(http.get(`*/api/${version}/denied`, () => HttpResponse.json({ detail: 'Forbidden' }, { status: 403 })))
+ await expect(client.get('/denied')).rejects.toMatchObject({ detail: 'Forbidden' })
+ expect(useAuthStore.getState().token).toBe('valid-session')
+ expect(fetch).not.toHaveBeenCalled()
+})
+
+it('refreshes an expired session once and retries with the new token', async () => {
+ useAuthStore.setState({ token: 'expired' })
+ const fetch = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(JSON.stringify({ data: { access_token: 'renewed' } }), { status: 200 }))
+ server.use(http.get('*/api/v1/protected', ({ request }) => request.headers.get('Authorization') === 'Bearer renewed'
+ ? HttpResponse.json({ data: { ok: true } }) : HttpResponse.json({}, { status: 401 })))
+ await expect(apiClient.get('/protected')).resolves.toEqual({ ok: true })
+ expect(useAuthStore.getState().token).toBe('renewed')
+ expect(fetch).toHaveBeenCalledTimes(1)
+})
+
+it('clears login if the refreshed token is still rejected', async () => {
+ useAuthStore.setState({ token: 'expired' })
+ const fetch = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(JSON.stringify({ data: { access_token: 'rejected' } }), { status: 200 }))
+ server.use(http.get('*/api/v1/protected', () => HttpResponse.json({ detail: 'Unauthorized' }, { status: 401 })))
+ await expect(apiClient.get('/protected')).rejects.toMatchObject({ detail: 'Unauthorized' })
+ expect(useAuthStore.getState().token).toBeNull()
+ expect(fetch).toHaveBeenCalledTimes(1)
+})
diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts
index 2e7b5700..ea6b5e1e 100644
--- a/frontend/src/api/client.ts
+++ b/frontend/src/api/client.ts
@@ -65,7 +65,7 @@ function createApiClient(baseURL: string, options?: { runtimeDelegation?: boolea
return client(config)
}
}
- if (!options?.runtimeDelegation && !options?.preserveSessionOnAuthorizationFailure && (status === 401 || status === 403)) {
+ if (!options?.runtimeDelegation && !options?.preserveSessionOnAuthorizationFailure && status === 401) {
useAuthStore.getState().logout()
}
if (options?.runtimeDelegation && status === 401) {
diff --git a/frontend/src/i18n/en.json b/frontend/src/i18n/en.json
index 6ab298cf..9b6e9412 100644
--- a/frontend/src/i18n/en.json
+++ b/frontend/src/i18n/en.json
@@ -124,8 +124,8 @@
"confirm_delete_msg": "Delete ontology \"{{name}}\"? This cannot be undone.",
"cancel": "Cancel",
"confirm": "Delete",
- "delete_failed_bound_agent": "Cannot delete: this ontology is bound to an Agent — unbind it first",
- "delete_failed_dependent_data": "Cannot delete: this ontology has dependent execution records (runs, sandbox simulations, etc.) — clean those up first",
+ "delete_failed_bound_agent": "Cannot delete: this ontology is bound to an Agent version, and Agent version bindings can never be changed once created — so this ontology cannot be deleted",
+ "delete_failed_dependent_data": "Cannot delete: this ontology has published releases with dependent execution records (runs, sandbox simulations, etc.) that are kept permanently for audit purposes — so this ontology cannot be deleted",
"delete_failed_generic": "Delete failed, please try again",
"deleting": "Deleting…",
"select_all": "Select all",
@@ -247,7 +247,6 @@
"delete_title": "Delete Rule",
"delete_confirm": "Delete rule “{{name}}”? This cannot be undone.",
"ph_search": "Search rule name / definition…",
- "publish_all": "Publish All",
"col_name": "Name",
"col_definition": "Definition",
"col_desc": "Description",
@@ -263,7 +262,6 @@
"empty": "No actions yet",
"delete_title": "Delete Action",
"delete_confirm": "Delete action “{{name}}”? This cannot be undone.",
- "publish_all": "Publish All",
"col_name": "Name",
"col_rule": "Rule",
"col_desc": "Description",
@@ -328,6 +326,8 @@
"empty": "No models configured. Click \"Add Model\" to get started.",
"confirm_delete": "Confirm Delete",
"confirm_delete_msg": "Delete model \"{{name}}\"?",
+ "delete_failed_referenced": "Cannot delete: this model is still used by an Agent. Unbind or delete the Agent first.",
+ "delete_failed_generic": "Delete failed, please try again",
"config_type_llm": "LLM Config",
"config_type_ocr": "OCR Config",
"config_type_other": "Other Config",
@@ -565,6 +565,7 @@
"tools": {
"available_later": "Available later",
"bind": "Bind",
+ "blocked_hint": "Your account has no data grant (read schema, instances and relations) for: {{names}}. Ask an admin to grant data access.",
"bound": "Bound",
"capabilities": "Capabilities",
"external": "External Tools",
@@ -1306,7 +1307,15 @@
"test_failed": "Failed",
"test_testing": "Testing...",
"test_connection": "Test Connection",
- "sync_mode_label": "Sync Mode"
+ "sync_mode_label": "Sync Mode",
+ "saved_connection_label": "Saved Connection",
+ "source_table_label": "Source Table",
+ "select_connection": "Select a saved connection",
+ "select_table": "Select a table",
+ "loading_connections": "Loading connections...",
+ "loading_resources": "Loading tables...",
+ "no_saved_connections": "Create a database connection in Data Connections first",
+ "resources_failed": "Could not discover tables"
},
"outputInspector": {
"output_type_label": "Output Type",
@@ -1411,7 +1420,16 @@
"drop_release": "Release to add files",
"drop_hint": "Drag files here, or",
"drop_hint_click": "click to select",
- "drop_formats_hint": "Supports CSV, XLSX, JSON, PDF, DOCX and more, multiple selection allowed"
+ "drop_formats_hint": "Supports CSV, XLSX, JSON, PDF, DOCX and more, multiple selection allowed",
+ "test_connection": "Test Connection",
+ "testing": "Testing...",
+ "test_success": "Connection succeeded",
+ "test_failed": "Connection failed",
+ "discover_tables": "Discover Tables",
+ "hide_tables": "Hide Tables",
+ "no_tables": "No tables found",
+ "resources_failed": "Could not discover tables",
+ "pipeline_sync_hint": "Run a selected table from Pipeline Builder"
},
"curatedReview": {
"notes_ph": "Review notes (optional)",
diff --git a/frontend/src/i18n/zh.json b/frontend/src/i18n/zh.json
index ad9a4afe..29ca657f 100644
--- a/frontend/src/i18n/zh.json
+++ b/frontend/src/i18n/zh.json
@@ -124,8 +124,8 @@
"confirm_delete_msg": "确定要删除本体「{{name}}」吗?此操作不可撤销。",
"cancel": "取消",
"confirm": "确认删除",
- "delete_failed_bound_agent": "无法删除:该本体已绑定至智能体,请先解绑对应智能体后再删除",
- "delete_failed_dependent_data": "无法删除:该本体存在关联的执行记录(如运行历史、沙盒模拟等),请先清理后再删除",
+ "delete_failed_bound_agent": "无法删除:该本体已被某个智能体版本绑定,智能体版本一经创建绑定即不可更改,因此这个本体无法删除",
+ "delete_failed_dependent_data": "无法删除:该本体存在已发布版本及关联的执行记录(运行历史、沙盒模拟等),出于审计要求这些记录会永久保留,因此本体无法删除",
"delete_failed_generic": "删除失败,请稍后重试",
"deleting": "删除中…",
"select_all": "全选",
@@ -247,7 +247,6 @@
"delete_title": "删除规则",
"delete_confirm": "确定删除规则「{{name}}」吗?此操作不可撤销。",
"ph_search": "搜索规则名称 / 定义…",
- "publish_all": "发布全部",
"col_name": "名称",
"col_definition": "定义",
"col_desc": "描述",
@@ -263,7 +262,6 @@
"empty": "暂无动作",
"delete_title": "删除动作",
"delete_confirm": "确定删除动作「{{name}}」吗?此操作不可撤销。",
- "publish_all": "发布全部",
"col_name": "名称",
"col_rule": "规则",
"col_desc": "描述",
@@ -328,6 +326,8 @@
"empty": "暂无模型配置,点击「添加模型」开始",
"confirm_delete": "确认删除",
"confirm_delete_msg": "删除模型「{{name}}」?",
+ "delete_failed_referenced": "无法删除:该模型仍被智能体使用,请先解绑或删除相关智能体后再删除",
+ "delete_failed_generic": "删除失败,请稍后重试",
"config_type_llm": "LLM配置",
"config_type_ocr": "OCR配置",
"config_type_other": "其他配置",
@@ -565,6 +565,7 @@
"tools": {
"available_later": "后续提供",
"bind": "绑定",
+ "blocked_hint": "当前账号没有以下本体的数据授权(读取结构、实例和关系):{{names}}。请联系管理员开通数据授权。",
"bound": "已绑定",
"capabilities": "能力",
"external": "外部工具",
@@ -1306,7 +1307,15 @@
"test_failed": "失败",
"test_testing": "测试中...",
"test_connection": "测试连接",
- "sync_mode_label": "同步模式"
+ "sync_mode_label": "同步模式",
+ "saved_connection_label": "已保存连接",
+ "source_table_label": "源数据表",
+ "select_connection": "选择已保存的连接",
+ "select_table": "选择数据表",
+ "loading_connections": "正在加载连接...",
+ "loading_resources": "正在加载数据表...",
+ "no_saved_connections": "请先在数据连接中创建数据库连接",
+ "resources_failed": "无法发现数据表"
},
"outputInspector": {
"output_type_label": "输出类型",
@@ -1411,7 +1420,16 @@
"drop_release": "松开以添加文件",
"drop_hint": "拖拽文件到此处,或",
"drop_hint_click": "点击选择",
- "drop_formats_hint": "支持 CSV、XLSX、JSON、PDF、DOCX 等格式,可多选"
+ "drop_formats_hint": "支持 CSV、XLSX、JSON、PDF、DOCX 等格式,可多选",
+ "test_connection": "测试连接",
+ "testing": "测试中...",
+ "test_success": "连接成功",
+ "test_failed": "连接失败",
+ "discover_tables": "发现数据表",
+ "hide_tables": "收起数据表",
+ "no_tables": "未发现数据表",
+ "resources_failed": "无法发现数据表",
+ "pipeline_sync_hint": "请在 Pipeline Builder 中选择数据表并运行"
},
"curatedReview": {
"notes_ph": "审核备注(可选)",
diff --git a/frontend/src/pages/agents/application/ConversationPanel.test.tsx b/frontend/src/pages/agents/application/ConversationPanel.test.tsx
index 692c6eb8..8153f06b 100644
--- a/frontend/src/pages/agents/application/ConversationPanel.test.tsx
+++ b/frontend/src/pages/agents/application/ConversationPanel.test.tsx
@@ -14,6 +14,20 @@ function renderPanel(messages: AgentMessage[], stream: StreamState = initialStre
}
describe('P4A-CONVERSATION', () => {
+ it('hides prior-turn evidence synchronously when the turn changes', async () => {
+ const get = vi.spyOn(apiClient, 'get').mockImplementation(url => url.includes('/old/')
+ ? Promise.resolve({ items: [{ sequence: 1, event_type: 'final_response', payload: { audit_event_id: 'old-audit' } }] }) as never
+ : new Promise(() => {}))
+ const props = { messages: [], stream: { ...initialStreamState, terminal: true }, clarification: null,
+ pendingApprovalId: null, onSend: () => {}, onAnswerClarification: () => {}, onApprovalResolved: () => {}, onRetry: () => {} }
+ try {
+ const view = render(
{validation.blocked.join(', ')}
++ {t('agent.tools.blocked_hint', { + names: validation.blocked.map(id => ontologies.find(o => o.id === id)?.name ?? id).join(', '), + })} +
)} } - {showForm && ( + {canEdit && showForm && ({formError}
} + {testStatus !== 'idle' &&
+ {testStatus === 'success' ?
{c.name}
-{meta.labelKey ? t(meta.labelKey) : KIND_LABEL_FALLBACK[c.kind]}
+{c.name}
+{meta.labelKey ? t(meta.labelKey) : KIND_LABEL_FALLBACK[c.kind]}
+{resourceErrors[c.id]}
+ ) : resources[c.id]?.length ? ( +{t('connectionsTab.no_tables')}
} +