✨Feat: Add AIDP search tool https://github.com/ModelEngine-Group/nexent/issues/2788

backend/apps/aidp_app.py

@@ -0,0 +1,43 @@
+"""
+AIDP App Layer
+FastAPI endpoints for AIDP knowledge base list proxy.
+"""
+import logging
+from http import HTTPStatus
+from typing import Annotated
+
+from fastapi import APIRouter, Query
+from fastapi.responses import JSONResponse
+
+from consts.error_code import ErrorCode
+from consts.exceptions import AppException
+from services.aidp_service import fetch_aidp_knowledge_bases_impl
+
+router = APIRouter(prefix="/aidp")
+logger = logging.getLogger("aidp_app")
+
+
+@router.get("/knowledge-bases")
+async def fetch_aidp_knowledge_bases_api(
+    server_url: Annotated[str, Query(description="AIDP API server URL")],
+    api_key: Annotated[str, Query(description="AIDP API key")],
+    page: Annotated[int, Query(ge=1, description="Page number starting from 1")] = 1,
+    page_size: Annotated[int, Query(ge=1, le=100, description="Page size from 1 to 100")] = 20,
+) -> JSONResponse:
+    """Fetch paginated knowledge bases from the external AIDP API."""
+    try:
+        result = fetch_aidp_knowledge_bases_impl(
+            server_url=server_url,
+            api_key=api_key,
+            page=page,
+            page_size=page_size,
+        )
+        return JSONResponse(status_code=HTTPStatus.OK, content=result)
+    except AppException:
+        raise
+    except Exception as e:
+        logger.exception("Failed to fetch AIDP knowledge bases: %s", e)
+        raise AppException(
+            ErrorCode.AIDP_SERVICE_ERROR,
+            f"Failed to fetch AIDP knowledge bases: {str(e)}",
+        )

backend/apps/config_app.py

@@ -33,6 +33,7 @@
 from apps.monitoring_app import router as monitoring_router
 from apps.a2a_server_app import router as a2a_server_router
 from apps.haotian_app import router as haotian_router
+from apps.aidp_app import router as aidp_router
 from apps.cas_app import router as cas_router
 from consts.const import IS_SPEED_MODE
 from services.prompt_template_service import sync_system_default_prompt_template
@@ -92,3 +93,4 @@ async def sync_default_prompt_template_on_startup():
 app.include_router(a2a_client_router)
 app.include_router(a2a_server_router)
 app.include_router(haotian_router)
+app.include_router(aidp_router)

backend/consts/error_code.py

@@ -189,6 +189,12 @@ class ErrorCode(Enum):
     IDATA_RATE_LIMIT = "130405"  # iData rate limit
     IDATA_RESPONSE_ERROR = "130406"  # iData response error
 
+    # 05 - AIDP Service
+    AIDP_SERVICE_ERROR = "130501"  # AIDP service error
+    AIDP_CONFIG_INVALID = "130502"  # Invalid AIDP configuration
+    AIDP_CONNECTION_ERROR = "130503"  # AIDP connection error
+    AIDP_AUTH_ERROR = "130504"  # AIDP auth error
+
     # ==================== 14 Northbound / 北向接口 ====================
     # 01 - Request
     NORTHBOUND_REQUEST_FAILED = "140101"  # Northbound request failed
@@ -254,6 +260,10 @@ class ErrorCode(Enum):
     ErrorCode.IDATA_CONNECTION_ERROR: 502,
     ErrorCode.IDATA_RESPONSE_ERROR: 502,
     ErrorCode.IDATA_RATE_LIMIT: 429,
+    # AIDP (module 13)
+    ErrorCode.AIDP_CONFIG_INVALID: 400,
+    ErrorCode.AIDP_AUTH_ERROR: 401,
+    ErrorCode.AIDP_CONNECTION_ERROR: 502,
     # OAuth (module 16)
     ErrorCode.OAUTH_PROVIDER_NOT_CONFIGURED: 400,
     ErrorCode.OAUTH_PROVIDER_DISABLED: 400,

backend/consts/error_message.py

@@ -123,6 +123,16 @@ class ErrorMessage:
         ErrorCode.DIFY_AUTH_ERROR: "Dify authentication failed. Please check your API key.",
         ErrorCode.DIFY_RATE_LIMIT: "Dify API rate limit exceeded. Please try again later.",
         ErrorCode.ME_CONNECTION_FAILED: "Failed to connect to ME service.",
+        ErrorCode.IDATA_SERVICE_ERROR: "iData service error.",
+        ErrorCode.IDATA_CONFIG_INVALID: "iData configuration invalid. Please check URL and API key format.",
+        ErrorCode.IDATA_CONNECTION_ERROR: "Failed to connect to iData. Please check network connection and URL.",
+        ErrorCode.IDATA_RESPONSE_ERROR: "Failed to parse iData response. Please check API URL.",
+        ErrorCode.IDATA_AUTH_ERROR: "iData authentication failed. Please check your API key.",
+        ErrorCode.IDATA_RATE_LIMIT: "iData API rate limit exceeded. Please try again later.",
+        ErrorCode.AIDP_SERVICE_ERROR: "AIDP service error.",
+        ErrorCode.AIDP_CONFIG_INVALID: "AIDP configuration invalid. Please check URL and API key format.",
+        ErrorCode.AIDP_CONNECTION_ERROR: "Failed to connect to AIDP. Please check network connection and URL.",
+        ErrorCode.AIDP_AUTH_ERROR: "AIDP authentication failed. Please check your API key.",
 
         # ==================== 14 Northbound / 北向接口 ====================
         ErrorCode.NORTHBOUND_REQUEST_FAILED: "Northbound request failed.",

backend/database/conversation_db.py

@@ -623,9 +623,18 @@ def get_conversation_history(conversation_id: int, user_id: Optional[str] = None
         }
 
 
+def _image_exists(session, message_id: int, image_url: str) -> bool:
+    stmt = select(ConversationSourceImage).where(
+        ConversationSourceImage.message_id == message_id,
+        ConversationSourceImage.image_url == image_url,
+        ConversationSourceImage.delete_flag == 'N'
+    ).limit(1)
+    return session.execute(stmt).scalar_one_or_none() is not None
+
+
 def create_source_image(image_data: Dict[str, Any], user_id: Optional[str] = None) -> int:
     """
-    Create image source reference
+    Create image source reference (skips if the same message_id + image_url already exists).
 
     Args:
         image_data: Dictionary containing image data, must include the following fields:
@@ -634,17 +643,22 @@ def create_source_image(image_data: Dict[str, Any], user_id: Optional[str] = Non
         user_id: Reserved parameter for created_by and updated_by fields
 
     Returns:
-        int: Newly created image ID (auto-increment ID)
+        int: Newly created image ID (auto-increment ID), or -1 if skipped due to duplicate
     """
     with get_db_session() as session:
         # Ensure message_id is of integer type
         message_id = int(image_data['message_id'])
+        image_url = image_data['image_url']
+
+        # Skip duplicate: same message_id + image_url already in DB
+        if _image_exists(session, message_id, image_url):
+            return -1
 
         # Prepare data dictionary
         data = {
             "message_id": message_id,
             "conversation_id": image_data.get('conversation_id'),
-            "image_url": image_data['image_url'],
+            "image_url": image_url,
             "delete_flag": 'N',
             # Use the database's CURRENT_TIMESTAMP function
             "create_time": func.current_timestamp()

backend/services/aidp_service.py

@@ -0,0 +1,99 @@
+"""
+AIDP Service Layer
+Handles API calls to AIDP for paginated knowledge base listing.
+"""
+import logging
+from typing import Any, Dict
+from urllib.parse import urljoin
+
+import httpx
+
+from consts.error_code import ErrorCode
+from consts.exceptions import AppException
+from nexent.utils.http_client_manager import http_client_manager
+
+logger = logging.getLogger("aidp_service")
+
+_LIST_PATH = "/KnowledgeBase/Tenants/aidp/KnowledgeBases"
+
+
+def _validate_params(server_url: str, api_key: str) -> str:
+    """Validate parameters and return normalized base URL."""
+    if not server_url or not isinstance(server_url, str):
+        raise AppException(
+            ErrorCode.AIDP_CONFIG_INVALID,
+            "AIDP server_url is required and must be a non-empty string",
+        )
+    if not server_url.startswith(("http://", "https://")):
+        raise AppException(
+            ErrorCode.AIDP_CONFIG_INVALID,
+            "AIDP server_url must start with http:// or https://",
+        )
+    if not api_key or not isinstance(api_key, str):
+        raise AppException(
+            ErrorCode.AIDP_CONFIG_INVALID,
+            "AIDP api_key is required and must be a non-empty string",
+        )
+    return server_url.rstrip("/")
+
+
+def fetch_aidp_knowledge_bases_impl(
+    server_url: str,
+    api_key: str,
+    page: int = 1,
+    page_size: int = 20,
+) -> Dict[str, Any]:
+    """Fetch paginated knowledge bases from AIDP API."""
+    normalized_url = _validate_params(server_url, api_key)
+
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+
+    list_path = f"{_LIST_PATH}?page={page}&page_size={page_size}"
+    list_url = urljoin(f"{normalized_url}/", list_path)
+    logger.info("Fetching AIDP knowledge bases from %s", list_url)
+
+    try:
+        client = http_client_manager.get_sync_client(
+            base_url=normalized_url,
+            timeout=20.0,
+            verify_ssl=True,
+        )
+        response = client.get(list_url, headers=headers)
+        response.raise_for_status()
+        result = response.json()
+        if not isinstance(result, dict):
+            raise AppException(
+                ErrorCode.AIDP_SERVICE_ERROR,
+                "Unexpected AIDP knowledge base response format",
+            )
+        return result
+    except httpx.RequestError as e:
+        logger.exception("AIDP request failed: %s", e)
+        raise AppException(
+            ErrorCode.AIDP_CONNECTION_ERROR,
+            f"AIDP API request failed: {str(e)}",
+        )
+    except httpx.HTTPStatusError as e:
+        logger.exception(
+            "AIDP API HTTP error: %s, status_code: %s",
+            e,
+            e.response.status_code,
+        )
+        if e.response.status_code in (401, 403):
+            raise AppException(
+                ErrorCode.AIDP_AUTH_ERROR,
+                f"AIDP authentication failed: {str(e)}",
+            )
+        raise AppException(
+            ErrorCode.AIDP_SERVICE_ERROR,
+            f"AIDP API HTTP error {e.response.status_code}: {str(e)}",
+        )
+    except ValueError as e:
+        logger.exception("Failed to parse AIDP API response: %s", e)
+        raise AppException(
+            ErrorCode.AIDP_SERVICE_ERROR,
+            f"Failed to parse AIDP API response: {str(e)}",
+        )

backend/services/conversation_management_service.py

@@ -127,7 +127,15 @@ def save_message(request: MessageRequest, user_id: str, tenant_id: str):
                     # Parse image URL list
                     content_json = json.loads(unit_content)
                     if isinstance(content_json, dict) and 'images_url' in content_json:
+                        # Deduplicate image URLs before saving
+                        seen_urls = set()
+                        unique_urls = []
                         for image_url in content_json['images_url']:
+                            if image_url not in seen_urls:
+                                seen_urls.add(image_url)
+                                unique_urls.append(image_url)
+                        # Also deduplicate against any URLs already saved in this same message
+                        for image_url in unique_urls:
                             image_data = {'message_id': message_id, 'conversation_id': conversation_id,
                                           'image_url': image_url}
                             create_source_image(image_data)
@@ -448,13 +456,15 @@ def get_conversation_history_service(conversation_id: int, user_id: str) -> List
                 search_by_message[message_id] = []
             search_by_message[message_id].append(search_item)
 
-        # Collect image content - grouped by message_id
+        # Collect image content - grouped by message_id, with URL deduplication
         image_by_message = {}
         for record in history_data['image_records']:
             message_id = record['message_id']
             if message_id not in image_by_message:
                 image_by_message[message_id] = []
-            image_by_message[message_id].append(record['image_url'])
+            # Only add if not already present (by URL)
+            if record['image_url'] not in image_by_message[message_id]:
+                image_by_message[message_id].append(record['image_url'])
 
         # Sort by message index and build final message list, including images and search content
         messages = []