diff --git a/.github/workflows/auto-unit-test.yml b/.github/workflows/auto-unit-test.yml index dace8dab6..1595fc769 100644 --- a/.github/workflows/auto-unit-test.yml +++ b/.github/workflows/auto-unit-test.yml @@ -36,7 +36,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version: '3.10' - name: Install uv run: pip install --upgrade uv diff --git a/.github/workflows/sdk_publish.yml b/.github/workflows/sdk_publish.yml index 3cc413381..1e5759277 100644 --- a/.github/workflows/sdk_publish.yml +++ b/.github/workflows/sdk_publish.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.10' - name: Install build dependencies run: | diff --git a/.gitignore b/.gitignore index e0bac2b47..ec5b3a3f9 100644 --- a/.gitignore +++ b/.gitignore @@ -61,9 +61,4 @@ data/ sdk/benchmark/.env /docker/.env.bak -.venv - -.pytest-tmp -doc/mermaid - -.claude/skills/python-import-triage \ No newline at end of file +.venv \ No newline at end of file diff --git a/backend/adapters/__init__.py b/backend/adapters/__init__.py deleted file mode 100644 index ed46fc888..000000000 --- a/backend/adapters/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from adapters.exception import JiuwenSDKError, JiuwenSDKUnavailableError, NexentCapabilityError - -try: - from adapters.jiuwen_sdk_adapter import JiuwenSDKAdapter -except ModuleNotFoundError: - JiuwenSDKAdapter = None # type: ignore[assignment, misc] - -__all__ = [ - "JiuwenSDKError", - "JiuwenSDKUnavailableError", - "NexentCapabilityError", - "JiuwenSDKAdapter", -] diff --git a/backend/adapters/exception.py b/backend/adapters/exception.py deleted file mode 100644 index 63812d3af..000000000 --- a/backend/adapters/exception.py +++ /dev/null @@ -1,13 +0,0 @@ -class JiuwenSDKError(Exception): - """Jiuwen SDK 调用失败的通用异常""" - pass - - -class JiuwenSDKUnavailableError(JiuwenSDKError): - """Jiuwen SDK 不可用（依赖缺失或未启用）""" - pass - - -class NexentCapabilityError(Exception): - """nexent 原生模式不支持该能力""" - pass diff --git a/backend/adapters/jiuwen_sdk_adapter.py b/backend/adapters/jiuwen_sdk_adapter.py deleted file mode 100644 index f62ce9d06..000000000 --- a/backend/adapters/jiuwen_sdk_adapter.py +++ /dev/null @@ -1,514 +0,0 @@ -""" -openjiuwen SDK adapter for Nexent. - -This module must be imported lazily (not at module load time) because -openjiuwen 0.1.13 has circular import bugs in its __init__.py files that -prevent the SDK from loading unless we bypass them. - -Import flow: - backend/adapters/__init__.py -> try/except -> JiuwenSDKAdapter = None - -> when needed: _install_jiuwen_bypasser() -> openjiuwen imports work -""" -import asyncio -import importlib.abc -import importlib.machinery -import json -import logging -import os -import sys -import types -from typing import Any, List, Literal, Optional - -logger = logging.getLogger("jiuwen_adapter") - -from adapters.exception import JiuwenSDKError - - -# ---------------------------------------------------------------------- -# Circular import bypasser for openjiuwen 0.1.13 -# -# openjiuwen has broken __init__.py files that create circular import chains: -# tune/__init__.py -> tune.optimizer -> core.operator -> agent_evolving -> ... -# This bypasser prevents those __init__.py files from executing while still -# allowing regular .py submodule files to load normally. -# ---------------------------------------------------------------------- -_CIRCULAR_CHAIN = { - "openjiuwen.agent_evolving", - "openjiuwen.agent_evolving.trainer", - "openjiuwen.agent_evolving.trainer.trainer", - "openjiuwen.agent_evolving.trainer.progress", - "openjiuwen.core", - "openjiuwen.dev_tools", - "openjiuwen.dev_tools.tune", - "openjiuwen.dev_tools.tune.optimizer", - "openjiuwen.dev_tools.tune.optimizer.instruction_optimizer", - "openjiuwen.dev_tools.prompt_builder", - "openjiuwen.dev_tools.prompt_builder.builder", -} - - -class _JiuwenInitBypasser(importlib.abc.MetaPathFinder, importlib.abc.Loader): - """ - Meta path finder that intercepts __init__.py loading within openjiuwen, - blocking only the packages in the circular import chain while letting - all other modules (including base.py files) load normally. - """ - - def find_spec(self, fullname: str, path: Any, target: Any = None) -> Any: - if not fullname.startswith("openjiuwen") or fullname == "openjiuwen": - return None - - try: - import openjiuwen as _oj - - pkg_root = _oj.__path__[0] - except ImportError: - return None - - parts = fullname.split(".")[1:] - file_path = pkg_root - for p in parts: - file_path = os.path.join(file_path, p) - - is_package = os.path.isdir(file_path) - if not is_package: - return None - - init_path = os.path.join(file_path, "__init__.py") - if not os.path.exists(init_path): - return None - - if fullname not in _CIRCULAR_CHAIN: - return None - - spec = importlib.machinery.ModuleSpec( - fullname, self, is_package=True, origin="" - ) - spec.submodule_search_locations = [file_path] - return spec - - def create_module(self, module: Any) -> None: - return None - - def exec_module(self, module: Any) -> None: - import openjiuwen as _oj - - pkg_root = _oj.__path__[0] - parts = module.__name__.split(".")[1:] - file_path = pkg_root - for p in parts: - file_path = os.path.join(file_path, p) - module.__path__ = [file_path] - module.__file__ = os.path.join(file_path, "__init__.py") - - def __getattr__(self, name: str) -> Any: - """Handle special attributes like find_distributions to prevent recursion.""" - import openjiuwen as _oj - import importlib - - # Prevent recursion when Python scans sys.meta_path for find_distributions etc. - if name in ( - "find_distributions", - "find_module", - "__path__", - "__name__", - "__file__", - "__loader__", - "__package__", - "__spec__", - ): - raise AttributeError(name) - - pkg_root = _oj.__path__[0] - parts = self.__name__.split(".")[1:] + [name] - file_path = pkg_root - for p in parts: - file_path = os.path.join(file_path, p) - - # If it's a package directory, import it as a submodule - if os.path.isdir(file_path) and os.path.exists(os.path.join(file_path, "__init__.py")): - return importlib.import_module(f"{self.__name__}.{name}") - # If it's a regular .py file - if os.path.exists(file_path + ".py"): - return importlib.import_module(f"{self.__name__}.{name}") - raise AttributeError(name) - - -_bypasser_installed = False - - -def _install_jiuwen_bypasser() -> bool: - """ - Install the circular import bypasser for openjiuwen. - Returns True if installed, False if already installed or openjiuwen not available. - """ - global _bypasser_installed - if _bypasser_installed: - return True - - # Stub missing optional dependencies before openjiuwen import chain reaches them - _stubbed = [ - ("pymilvus", {"is_successful": lambda *args, **kwargs: True}), - ("dashscope", {}), - ("pdfplumber", {}), - ] - for _name, _attrs in _stubbed: - if _name not in sys.modules: - _mod = types.ModuleType(_name) - for _k, _v in _attrs.items(): - setattr(_mod, _k, _v) - sys.modules[_name] = _mod - _mod.__path__ = [] - - # Pre-create nested stub modules for pymilvus.client.utils chain - if "pymilvus.client" not in sys.modules: - _client_mod = types.ModuleType("pymilvus.client") - _client_mod.__path__ = [] - sys.modules["pymilvus.client"] = _client_mod - if "pymilvus.client.utils" not in sys.modules: - _utils_mod = types.ModuleType("pymilvus.client.utils") - _utils_mod.is_successful = lambda *args, **kwargs: True - sys.modules["pymilvus.client.utils"] = _utils_mod - - # Stub dashscope sub-modules that may be imported lazily - _dashscope_subs = [ - ("dashscope.api_entities", {}), - ("dashscope.api_entities.data", {}), - ("dashscope.api_entities.dashscope_response", {"DashScopeAPIResponse": object}), - ("dashscope.common", {"REQUEST_TIMEOUT_KEYWORD": "timeout"}), - ("dashscope.common.constants", {"REQUEST_TIMEOUT_KEYWORD": "timeout"}), - ] - for _name, _attrs in _dashscope_subs: - if _name not in sys.modules: - _m = types.ModuleType(_name) - _m.__path__ = [] - for _k, _v in _attrs.items(): - setattr(_m, _k, _v) - sys.modules[_name] = _m - - try: - import openjiuwen # noqa: F401 - except ImportError: - return False - - for finder in sys.meta_path: - if isinstance(finder, _JiuwenInitBypasser): - _bypasser_installed = True - return True - - sys.meta_path.insert(0, _JiuwenInitBypasser()) - _bypasser_installed = True - return True - - -# ---------------------------------------------------------------------- -# Language helpers -# ---------------------------------------------------------------------- -LANGUAGE_MAP = {"zh": "zh-CN", "en": "en-US"} - - -def normalize_language(language: str) -> str: - return LANGUAGE_MAP.get(language, "zh-CN") - - -def run_async(coro): - """ - Safely run async coroutine from sync context (FastAPI or Celery). - Handles existing event loops properly. - """ - try: - loop = asyncio.get_running_loop() - except RuntimeError: - return asyncio.run(coro) - - if loop.is_running(): - try: - import nest_asyncio - nest_asyncio.apply() - return loop.run_until_complete(coro) - except ImportError: - import concurrent.futures - - def run_in_thread(): - new_loop = asyncio.new_event_loop() - asyncio.set_event_loop(new_loop) - try: - return new_loop.run_until_complete(coro) - finally: - new_loop.close() - - with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: - future = executor.submit(run_in_thread) - return future.result() - - return loop.run_until_complete(coro) - - -# ---------------------------------------------------------------------- -# Jiuwen SDK lazy import helpers -# ---------------------------------------------------------------------- -def _lazy_import_jiuwen_config(): - """Lazily import only lightweight Jiuwen config classes.""" - _install_jiuwen_bypasser() - - try: - import openjiuwen # noqa: F401 - except ImportError as e: - raise JiuwenSDKError(f"Jiuwen SDK 未安装: {e}") from e - - from openjiuwen.core.foundation.llm.schema.config import ( - ModelRequestConfig, - ModelClientConfig, - ProviderType, - ) - - return ModelRequestConfig, ModelClientConfig, ProviderType - - -def build_jiuwen_model_configs(model_id: int, tenant_id: str): - """将 nexent 模型配置转换为 Jiuwen 配置对象""" - from database.model_management_db import get_model_by_model_id - from utils.config_utils import get_model_name_from_config - - ModelRequestConfig, ModelClientConfig, ProviderType = _lazy_import_jiuwen_config() - - model_config = get_model_by_model_id(model_id, tenant_id) - if not model_config: - raise JiuwenSDKError(f"model_id={model_id} not found") - - api_base = (model_config.get("base_url", "") or "").strip() - if not api_base: - api_base = "https://api.openai.com/v1" - - # Jiuwen ModelClientConfig defaults to timeout=60.0, max_retries=3. - # For prompt optimization calls, 60s can be too small. Reuse Nexent model config timeout_seconds. - timeout_seconds = model_config.get("timeout_seconds") - if timeout_seconds is None: - timeout_seconds = 120 - - ssl_cert = model_config.get("ssl_cert") or None - ssl_verify = model_config.get("ssl_verify", True) - if ssl_verify and not ssl_cert: - ssl_verify = False - - client_config = ModelClientConfig( - client_provider=ProviderType.OpenAI, - api_key=model_config["api_key"], - api_base=api_base, - timeout=float(timeout_seconds), - verify_ssl=ssl_verify, - ssl_cert=ssl_cert, - ) - - request_config = ModelRequestConfig( - model_name=get_model_name_from_config(model_config), - temperature=0.3, - ) - return request_config, client_config - - -def _lazy_import_jiuwen_builders(): - """Lazily import prompt builders only when optimization paths need them.""" - _install_jiuwen_bypasser() - - try: - import openjiuwen # noqa: F401 - except ImportError as e: - raise JiuwenSDKError(f"Jiuwen SDK 未安装: {e}") from e - - from openjiuwen.dev_tools.prompt_builder.builder.feedback_prompt_builder import ( - FeedbackPromptBuilder, - ) - from openjiuwen.dev_tools.prompt_builder.builder.badcase_prompt_builder import ( - BadCasePromptBuilder, - ) - - return FeedbackPromptBuilder, BadCasePromptBuilder - - -def _unwrap_prompt_response(text: str) -> str: - """Strip JSON wrapper or markdown fence that Jiuwen LLM sometimes generates.""" - _logger = logging.getLogger("jiuwen_adapter") - _logger.debug(f"[unwrap] raw ({len(text)} chars): {text[:200]}") - - # Step 1: strip markdown code fences - text = text.strip() - if text.startswith("```"): - for lang in ("json", ""): - fence = f"```{lang}\n" - if text.startswith(fence): - text = text[len(fence):] - if text.endswith("\n```"): - text = text[:-4] - elif text.endswith("```"): - text = text[:-3] - break - text = text.strip() - _logger.debug(f"[unwrap] after fence strip ({len(text)} chars)") - - # Step 2: try standard JSON parse (handles format 1 and 2) - if text.startswith("{"): - try: - parsed = json.loads(text) - if isinstance(parsed, dict) and "prompt" in parsed: - result = parsed["prompt"].strip() - _logger.debug(f"[unwrap] extracted prompt ({len(result)} chars)") - return result - if isinstance(parsed, dict) and "result" in parsed: - result = parsed["result"].strip() - _logger.debug(f"[unwrap] extracted result ({len(result)} chars)") - return result - except Exception: - pass - - # Step 3: format 3 and 4 - raw text (possibly multi-line), return as-is - _logger.debug(f"[unwrap] no JSON wrapper, returning raw ({len(text)} chars)") - return text - - -def _lazy_import_jiuwen_tune_types(): - """Lazily import Jiuwen tune types only when badcase flow needs them.""" - _install_jiuwen_bypasser() - from openjiuwen.dev_tools.tune.base import Case, EvaluatedCase - return Case, EvaluatedCase - - -def to_jiuwen_evaluated_case(bad_case) -> Any: - """将 nexent BadCase 转换为 Jiuwen EvaluatedCase""" - Case, EvaluatedCase = _lazy_import_jiuwen_tune_types() - - case = Case( - inputs={"question": bad_case.question}, - label={"answer": bad_case.label or ""}, - ) - return EvaluatedCase( - case=case, - answer={"content": bad_case.answer}, - score=0.0, - reason=bad_case.reason or "", - ) - - -# ---------------------------------------------------------------------- -# Main adapter class -# ---------------------------------------------------------------------- -class JiuwenSDKAdapter: - """ - Jiuwen SDK 调用适配器 - - 封装 Jiuwen SDK 的所有调用，内部不处理降级， - 失败时抛出 JiuwenSDKError，由上层 PromptOptimizationService 决定是否降级 - """ - - def __init__(self, model_id: int, tenant_id: str): - self.model_id = model_id - self.tenant_id = tenant_id - self.logger = logging.getLogger("jiuwen_adapter") - - def _ensure_available(self): - """确保 Jiuwen SDK 可用""" - if not _bypasser_installed: - _install_jiuwen_bypasser() - - try: - import openjiuwen # noqa: F401 - except ImportError as e: - raise JiuwenSDKError(f"Jiuwen SDK 未安装: {e}") from e - - def optimize( - self, - prompt: str, - feedback: str, - mode: Literal["general", "insert", "select"] = "general", - start_pos: Optional[int] = None, - end_pos: Optional[int] = None, - language: str = "zh", - ) -> str: - """ - 调用 Jiuwen FeedbackPromptBuilder - - Raises: - JiuwenSDKError: SDK 调用失败 - """ - self._ensure_available() - - logger.info(f"[jiuwen-adapter] mode={mode}, start_pos={start_pos}, end_pos={end_pos}") - - request_config, client_config = build_jiuwen_model_configs( - self.model_id, self.tenant_id - ) - logger.info( - f"[jiuwen-adapter] model_id={self.model_id}, tenant_id={self.tenant_id}, " - f"api_base={client_config.api_base}, model={request_config.model_name}, " - f"timeout={getattr(client_config, 'timeout', None)}, max_retries={getattr(client_config, 'max_retries', None)}" - ) - FeedbackPromptBuilder, _ = _lazy_import_jiuwen_builders() - - builder = FeedbackPromptBuilder( - model_config=request_config, - model_client_config=client_config, - ) - - try: - result = run_async( - builder.build( - prompt=prompt, - feedback=feedback, - mode=mode, - start_pos=start_pos, - end_pos=end_pos, - language=normalize_language(language), - ) - ) - if result is None: - raise JiuwenSDKError("Jiuwen FeedbackPromptBuilder 返回为空") - return _unwrap_prompt_response(str(result)) - except Exception as e: - self.logger.error(f"Jiuwen FeedbackPromptBuilder 调用失败: {e}") - raise JiuwenSDKError(f"优化调用失败: {e}") from e - - def optimize_badcase( - self, - prompt: str, - bad_cases: List, - language: str = "zh", - ) -> str: - """ - 调用 Jiuwen BadCasePromptBuilder - - Raises: - JiuwenSDKError: SDK 调用失败 - """ - self._ensure_available() - - _, BadCasePromptBuilder = _lazy_import_jiuwen_builders() - - request_config, client_config = build_jiuwen_model_configs( - self.model_id, self.tenant_id - ) - builder = BadCasePromptBuilder( - model_config=request_config, - model_client_config=client_config, - ) - - jiuwen_cases = [to_jiuwen_evaluated_case(bc) for bc in bad_cases] - - try: - result = run_async( - builder.build( - prompt=prompt, - cases=jiuwen_cases, - language=normalize_language(language), - ) - ) - if result is None: - raise JiuwenSDKError("Jiuwen BadCasePromptBuilder 返回为空") - return _unwrap_prompt_response(str(result)) - except Exception as e: - self.logger.error(f"Jiuwen BadCasePromptBuilder 调用失败: {e}") - raise JiuwenSDKError(f"BadCasePromptBuilder 调用失败: {e}") from e - - def generate(self, **kwargs) -> dict: - """调用 Jiuwen 提示词生成能力""" - self._ensure_available() - raise JiuwenSDKError("Jiuwen 提示词生成能力尚未实现") diff --git a/backend/agents/create_agent_info.py b/backend/agents/create_agent_info.py index 7e3b42e28..50df7eb99 100644 --- a/backend/agents/create_agent_info.py +++ b/backend/agents/create_agent_info.py @@ -1,12 +1,12 @@ -import json -import threading +import threading import logging -from typing import Any, Dict, List, Optional +from typing import List, Optional from urllib.parse import urljoin +from datetime import datetime from jinja2 import Template, StrictUndefined from nexent.core.utils.observer import MessageObserver -from nexent.core.agents.agent_model import AgentRunInfo, ModelConfig, AgentConfig, ToolConfig, ExternalA2AAgentConfig, AgentHistory, AgentVerificationConfig +from nexent.core.agents.agent_model import AgentRunInfo, ModelConfig, AgentConfig, ToolConfig, ExternalA2AAgentConfig, AgentHistory from nexent.core.agents.agent_context import ContextManagerConfig from nexent.memory.memory_service import search_memory_in_levels @@ -22,11 +22,7 @@ from database.a2a_agent_db import PROTOCOL_JSONRPC from services.memory_config_service import build_memory_context from services.image_service import get_video_understanding_model, get_vlm_model -from database.agent_db import ( - search_agent_info_by_agent_id, - query_sub_agent_relations, - resolve_sub_agent_version_no, -) +from database.agent_db import search_agent_info_by_agent_id, query_sub_agents_id_list from database.agent_version_db import query_current_version_no from database.tool_db import search_tools_for_sub_agent from database.model_management_db import get_model_records, get_model_by_model_id @@ -37,71 +33,12 @@ from utils.config_utils import tenant_config_manager, get_model_name_from_config from utils.context_utils import build_context_components from consts.const import LOCAL_MCP_SERVER, MODEL_CONFIG_MAPPING, LANGUAGE, DATA_PROCESS_SERVICE, MINIO_DEFAULT_BUCKET -from consts.model import AgentToolParamsRequest, ToolParamsRequest from consts.exceptions import ValidationError logger = logging.getLogger("create_agent_info") logger.setLevel(logging.DEBUG) -def _normalize_tool_params_request(tool_params: Optional[ToolParamsRequest | Dict[str, Any]]) -> ToolParamsRequest: - """Normalize request-scoped tool parameter overrides into a ToolParamsRequest.""" - if tool_params is None: - return ToolParamsRequest() - if isinstance(tool_params, ToolParamsRequest): - return tool_params - if not isinstance(tool_params, dict): - raise ValidationError("tool_params must be an object.") - try: - return ToolParamsRequest.model_validate(tool_params) - except Exception as exc: - raise ValidationError(f"Invalid tool_params payload: {exc}") from exc - - -def _get_agent_tool_overrides( - tool_params: Optional[ToolParamsRequest], - agent_name: Optional[str], -) -> Dict[str, Dict[str, Any]]: - """Resolve tool overrides for a specific agent by its name.""" - if tool_params is None: - return {} - if not agent_name: - return {} - agent_override = tool_params.agents.get(agent_name) - if agent_override is None: - return {} - return dict(agent_override.tools) - - -def _merge_tool_params( - tool_record: Dict[str, Any], - override_params: Optional[Dict[str, Any]], - extra_params: Optional[Dict[str, Any]] = None, -) -> Dict[str, Any]: - """Merge request overrides on top of tool instance defaults from DB. - - Args: - tool_record: Tool configuration from database - override_params: Request-scoped overrides from tool_params - extra_params: Additional internal params not in DB schema (e.g., document_paths) - - Returns: - Merged params dict with DB defaults, overrides, and extra params - """ - merged_params: Dict[str, Any] = {} - for param in tool_record.get("params", []): - merged_params[param["name"]] = param.get("default") - - if override_params: - merged_params.update(override_params) - - # Extra params (e.g., internal access control params) always take precedence - if extra_params: - merged_params.update(extra_params) - - return merged_params - - def _build_internal_s3_url(file: dict) -> str: """Build a valid S3 URL for internal tools from uploaded file metadata.""" if not isinstance(file, dict): @@ -373,23 +310,18 @@ async def create_agent_config( allow_memory_search: bool = True, version_no: int = 0, override_model_id: int | None = None, - tool_params: Optional[ToolParamsRequest | Dict[str, Any]] = None, ): - normalized_tool_params = _normalize_tool_params_request(tool_params) agent_info = search_agent_info_by_agent_id( agent_id=agent_id, tenant_id=tenant_id, version_no=version_no) # create sub agent - sub_agent_relations = query_sub_agent_relations( + sub_agent_id_list = query_sub_agents_id_list( main_agent_id=agent_id, tenant_id=tenant_id, version_no=version_no) managed_agents = [] - for rel in sub_agent_relations: - sub_agent_id = rel['selected_agent_id'] - sub_agent_version_no = resolve_sub_agent_version_no( - selected_agent_id=sub_agent_id, - selected_agent_version_no=rel.get('selected_agent_version_no'), - tenant_id=tenant_id, - ) + for sub_agent_id in sub_agent_id_list: + # Get the current published version for this sub-agent (from draft version 0) + sub_agent_version_no = query_current_version_no( + agent_id=sub_agent_id, tenant_id=tenant_id) or 0 sub_agent_config = await create_agent_config( agent_id=sub_agent_id, tenant_id=tenant_id, @@ -399,20 +331,13 @@ async def create_agent_config( allow_memory_search=allow_memory_search, version_no=sub_agent_version_no, override_model_id=None, - tool_params=normalized_tool_params, ) managed_agents.append(sub_agent_config) # create external A2A agents (synchronous function, no await needed) external_a2a_agents = _get_external_a2a_agents(agent_id, tenant_id, version_no) - tool_list = await create_tool_config_list( - agent_id, - tenant_id, - user_id, - version_no=version_no, - tool_params=normalized_tool_params, - ) + tool_list = await create_tool_config_list(agent_id, tenant_id, user_id, version_no=version_no) # Build system prompt: prioritize segmented fields, fallback to original prompt field if not available duty_prompt = agent_info.get("duty_prompt", "") @@ -458,77 +383,6 @@ async def create_agent_config( # Bubble up to streaming layer so it can emit and fall back raise Exception(f"Failed to retrieve memory list: {e}") - # Append active memory tools if memory is enabled - if memory_context.user_config.memory_switch and memory_context.memory_config: - try: - memory_metadata = { - "memory_config": memory_context.memory_config, - "memory_user_config": memory_context.user_config, - "tenant_id": memory_context.tenant_id, - "user_id": memory_context.user_id, - "agent_id": memory_context.agent_id, - } - - store_tool_config = ToolConfig( - class_name="StoreMemoryTool", - name="store_memory", - description=( - "Save important information to long-term memory for future recall. " - "Use this when the user shares personal preferences, facts about themselves, " - "project context, or instructions that should persist across conversations. " - "Do NOT store transient information like temporary calculations, information " - "already in the knowledge base, or data the user explicitly says to forget." - ), - inputs=json.dumps({ - "content": { - "type": "string", - "description": "The information to remember", - "description_zh": "需要记住的信息" - } - }, ensure_ascii=False), - output_type="string", - params={}, - source="local", - usage=None, - metadata=memory_metadata, - ) - tool_list.append(store_tool_config) - - search_tool_config = ToolConfig( - class_name="SearchMemoryTool", - name="search_memory", - description=( - "Search long-term memory for relevant information from previous interactions. " - "Use this when you need context about the user's preferences, past decisions, " - "or previously discussed topics that aren't in the current conversation. " - "The system already provides some memory context automatically -- use this tool " - "when you need to search for specific information not already available." - ), - inputs=json.dumps({ - "query": { - "type": "string", - "description": "Natural language query describing what to search for", - "description_zh": "描述要搜索内容的自然语言查询" - }, - "top_k": { - "type": "integer", - "description": "Maximum number of results to return", - "description_zh": "返回结果的最大数量", - "default": 5, - "nullable": True - } - }, ensure_ascii=False), - output_type="string", - params={}, - source="local", - usage=None, - metadata=memory_metadata, - ) - tool_list.append(search_tool_config) - logger.debug("Active memory tools appended to agent tool list") - except Exception as e: - logger.warning(f"Failed to append active memory tools: {e}") - # Build knowledge base summary knowledge_base_summary = "" try: @@ -559,6 +413,7 @@ async def create_agent_config( # Get skills list for prompt template skills = _get_skills_for_template(agent_id, tenant_id, version_no) + time_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") is_manager = len(managed_agents) > 0 or len(external_a2a_agents) > 0 render_kwargs = { @@ -573,6 +428,7 @@ async def create_agent_config( "APP_DESCRIPTION": app_description, "memory_list": memory_list, "knowledge_base_summary": knowledge_base_summary, + "time": time_str, "user_id": user_id, } system_prompt = Template(prompt_template["system_prompt"], undefined=StrictUndefined).render(render_kwargs) @@ -601,6 +457,7 @@ async def create_agent_config( few_shots=few_shots_prompt, app_name=app_name, app_description=app_description, + time_str=time_str, user_id=user_id, language=language, is_manager=is_manager, @@ -633,48 +490,21 @@ async def create_agent_config( external_a2a_agents=external_a2a_agents, context_manager_config=cm_config, context_components=context_components, - verification_config=AgentVerificationConfig.model_validate(agent_info.get("verification_config") or {}), ) return agent_config -async def create_tool_config_list( - agent_id, - tenant_id, - user_id, - version_no: int = 0, - tool_params: Optional[ToolParamsRequest | Dict[str, Any]] = None, -): +async def create_tool_config_list(agent_id, tenant_id, user_id, version_no: int = 0): + # create tool tool_config_list = [] langchain_tools = await discover_langchain_tools() - normalized_tool_params = _normalize_tool_params_request(tool_params) # now only admin can modify the agent, user_id is not used tools_list = search_tools_for_sub_agent(agent_id, tenant_id, version_no=version_no) - - # Look up agent name for use in error messages. - # Agent name is optional for tool_params matching (matching uses tool identifiers only), - # but we include it in error messages so callers can identify which agent/tool caused a failure. - agent_info = search_agent_info_by_agent_id(agent_id=agent_id, tenant_id=tenant_id, version_no=version_no) - agent_name = agent_info.get("name") if agent_info else None - agent_tool_overrides = _get_agent_tool_overrides(normalized_tool_params, agent_name) - - tool_keys_seen = set() for tool in tools_list: - tool_identifier = tool.get("name") or tool.get("class_name") - if tool_identifier in tool_keys_seen: - raise ValidationError( - f"Duplicate tool identifier '{tool_identifier}' found in agent '{agent_name or agent_id}'." - ) - tool_keys_seen.add(tool_identifier) - - override_params = None - if tool.get("name") in agent_tool_overrides: - override_params = agent_tool_overrides[tool.get("name")] - elif tool.get("class_name") in agent_tool_overrides: - override_params = agent_tool_overrides[tool.get("class_name")] - - param_dict = _merge_tool_params(tool, override_params) + param_dict = {} + for param in tool.get("params", []): + param_dict[param["name"]] = param.get("default") tool_config = ToolConfig( class_name=tool.get("class_name"), name=tool.get("name"), @@ -693,21 +523,12 @@ async def create_tool_config_list( tool_config.metadata = langchain_tool break - # Extract document_paths for KnowledgeBaseSearchTool (internal access control, not in DB schema) - document_paths = None - if override_params and "document_paths" in override_params: - document_paths = override_params.get("document_paths") - # Also check using the tool name as key - if not document_paths: - kb_overrides = agent_tool_overrides.get("knowledge_base_search") - if kb_overrides and "document_paths" in kb_overrides: - document_paths = kb_overrides.get("document_paths") - # special logic for search tools that may use reranking models if tool_config.class_name == "KnowledgeBaseSearchTool": - rerank = tool_config.params.get("rerank", False) - rerank_model_name = tool_config.params.get("rerank_model_name", "") + rerank = param_dict.get("rerank", False) + rerank_model_name = param_dict.get("rerank_model_name", "") rerank_model = None + is_multimodal = bool(tool_config.params.pop("multimodal", False)) if rerank and rerank_model_name: rerank_model = get_rerank_model( tenant_id=tenant_id, model_name=rerank_model_name @@ -715,7 +536,7 @@ async def create_tool_config_list( # Build display_name to index_name mapping for LLM parameter conversion # Also build reverse mapping (index_name -> display_name) for knowledge_base_summary - index_names = tool_config.params.get("index_names", []) + index_names = param_dict.get("index_names", []) display_name_to_index_map = {} index_name_to_display_map = {} if index_names: @@ -731,14 +552,12 @@ async def create_tool_config_list( "rerank_model": rerank_model, "display_name_to_index_map": display_name_to_index_map, "index_name_to_display_map": index_name_to_display_map, - # Internal access control: restrict results to specific document paths (path_or_urls) - "document_paths": document_paths, } + # Must have embedding model for knowledge base search if not index_names: raise ValidationError( - f"[{agent_name or agent_id}] knowledge_base_search tool requires index_names, " - f"but it is not configured in the agent and not provided via tool_params.") + "Embedding model is required for knowledge_base_search but index_names is empty") embedding_model, _, _ = get_embedding_model_by_index_name(tenant_id, index_names[0]) if not embedding_model: @@ -747,8 +566,8 @@ async def create_tool_config_list( f"Please configure an embedding model for this knowledge base.") tool_config.metadata["embedding_model"] = embedding_model elif tool_config.class_name in ["DifySearchTool", "DataMateSearchTool"]: - rerank = tool_config.params.get("rerank", False) - rerank_model_name = tool_config.params.get("rerank_model_name", "") + rerank = param_dict.get("rerank", False) + rerank_model_name = param_dict.get("rerank_model_name", "") rerank_model = None if rerank and rerank_model_name: rerank_model = get_rerank_model( @@ -1042,7 +861,6 @@ async def create_agent_run_info( is_debug: bool = False, override_version_no: int | None = None, override_model_id: int | None = None, - tool_params: Optional[ToolParamsRequest | Dict[str, Any]] = None, ): # Determine which version_no to use based on is_debug flag # If is_debug=false, use the current published version (current_version_no) @@ -1075,7 +893,7 @@ async def create_agent_run_info( if override_model_id is not None: create_config_kwargs["override_model_id"] = override_model_id - agent_config = await create_agent_config(**create_config_kwargs, tool_params=tool_params) + agent_config = await create_agent_config(**create_config_kwargs) remote_mcp_list = await get_remote_mcp_server_list(tenant_id=tenant_id, is_need_auth=True) default_mcp_url = urljoin(LOCAL_MCP_SERVER, "sse") diff --git a/backend/apps/agent_app.py b/backend/apps/agent_app.py index 87abbf9e8..e280ff422 100644 --- a/backend/apps/agent_app.py +++ b/backend/apps/agent_app.py @@ -195,6 +195,8 @@ async def export_agent_api(request: AgentIDRequest, authorization: Optional[str] "Content-Disposition": f"attachment; filename=\"{result.get('filename', 'agent_export.zip')}\"" } ) + if isinstance(result, str): + result = json.loads(result) return ConversationResponse(code=0, message="success", data=result) except Exception as e: logger.error(f"Agent export error: {str(e)}") @@ -619,5 +621,3 @@ async def list_published_agents_api( raise HTTPException( status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail="Published agents list error." ) - - diff --git a/backend/apps/agent_repository_app.py b/backend/apps/agent_repository_app.py deleted file mode 100644 index e9da2fde0..000000000 --- a/backend/apps/agent_repository_app.py +++ /dev/null @@ -1,134 +0,0 @@ -import logging -from http import HTTPStatus -from typing import Optional - -from fastapi import APIRouter, Body, Header, HTTPException, Query -from starlette.responses import JSONResponse - -from consts.exceptions import SkillDuplicateError, UnauthorizedError -from services.agent_repository_service import ( - create_agent_repository_listing_impl, - import_agent_from_repository_impl, - list_agent_repository_listings_impl, - update_agent_repository_status_impl, -) -from utils.auth_utils import get_current_user_id - -agent_repository_router = APIRouter(prefix="/repository/agent") -logger = logging.getLogger("agent_repository_app") - - -@agent_repository_router.get("") -async def list_agent_repository_listings_api( - status: Optional[str] = Query(None, description="Filter by listing status"), - authorization: str = Header(None), -): - """List all marketplace repository listings with optional status filter.""" - try: - get_current_user_id(authorization) - result = list_agent_repository_listings_impl(status=status) - return JSONResponse(status_code=HTTPStatus.OK, content=result) - except UnauthorizedError as e: - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(e)) - except ValueError as e: - raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(e)) - except Exception as e: - logger.error(f"List agent repository listings error: {str(e)}") - raise HTTPException( - status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail="List agent repository listings error.", - ) - - -@agent_repository_router.patch("/{agent_repository_id}/status") -async def update_agent_repository_status_api( - agent_repository_id: int, - status: str = Body( - ..., - embed=True, - description=( - "New status: NOT_SHARED (未共享) / PENDING_REVIEW (待审核) / " - "REJECTED (审核驳回) / SHARED (已共享)" - ), - ), - authorization: str = Header(None), -): - """Update marketplace repository listing status (share, unshare, approve, reject).""" - try: - user_id, _ = get_current_user_id(authorization) - result = update_agent_repository_status_impl( - agent_repository_id=agent_repository_id, - status=status, - user_id=user_id, - ) - return JSONResponse(status_code=HTTPStatus.OK, content=result) - except UnauthorizedError as e: - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(e)) - except ValueError as e: - raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(e)) - except Exception as e: - logger.error(f"Update agent repository status error: {str(e)}") - raise HTTPException( - status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail="Update agent repository status error.", - ) - - -@agent_repository_router.post("/{agent_id}/versions/{version_no}") -async def create_agent_repository_listing_api( - agent_id: int, - version_no: int, - authorization: str = Header(None), -): - """Create or update a marketplace repository listing from an agent version snapshot.""" - try: - user_id, tenant_id = get_current_user_id(authorization) - result = await create_agent_repository_listing_impl( - agent_id=agent_id, - tenant_id=tenant_id, - user_id=user_id, - version_no=version_no, - ) - return JSONResponse(status_code=HTTPStatus.OK, content=result) - except UnauthorizedError as e: - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(e)) - except ValueError as e: - raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(e)) - except Exception as e: - logger.error(f"Create agent repository listing error: {str(e)}") - raise HTTPException( - status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail="Create agent repository listing error.", - ) - - -@agent_repository_router.post("/{agent_repository_id}/import") -async def import_agent_from_repository_api( - agent_repository_id: int, - authorization: Optional[str] = Header(None), -): - """Import an agent tree from a marketplace repository listing into the current tenant.""" - try: - await import_agent_from_repository_impl( - agent_repository_id=agent_repository_id, - authorization=authorization, - ) - return JSONResponse(status_code=HTTPStatus.OK, content={}) - except UnauthorizedError as e: - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(e)) - except SkillDuplicateError as exc: - raise HTTPException( - status_code=HTTPStatus.CONFLICT, - detail={ - "type": "skill_duplicate", - "duplicate_skills": exc.duplicate_names, - }, - ) - except ValueError as e: - raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(e)) - except Exception as e: - logger.error(f"Import agent from repository error: {str(e)}") - raise HTTPException( - status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail="Import agent from repository error.", - ) diff --git a/backend/apps/app_factory.py b/backend/apps/app_factory.py index 02816cec1..219da5b82 100644 --- a/backend/apps/app_factory.py +++ b/backend/apps/app_factory.py @@ -101,16 +101,6 @@ async def generic_exception_handler(request, exc): if isinstance(exc, AppException): return await app_exception_handler(request, exc) - # Handle NexentCapabilityError with a friendly message - from adapters.exception import NexentCapabilityError as _NCE - - if isinstance(exc, _NCE): - logger.warning(f"NexentCapabilityError: {exc}") - return JSONResponse( - status_code=400, - content={"message": str(exc)}, - ) - logger.error(f"Generic Exception: {exc}") return JSONResponse( status_code=500, diff --git a/backend/apps/cas_app.py b/backend/apps/cas_app.py deleted file mode 100644 index dbf4815f8..000000000 --- a/backend/apps/cas_app.py +++ /dev/null @@ -1,156 +0,0 @@ -import html -import logging -from http import HTTPStatus -from typing import Optional -from urllib.parse import parse_qs, urlsplit - -from fastapi import APIRouter, HTTPException, Query, Request -from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse - -from services.cas_service import ( - CAS_SERVER_URL, - CasAuthenticationError, - build_login_url, - build_renew_url, - get_cas_config, - login_with_ticket, - renew_with_ticket, - revoke_from_logout_request, -) - -logger = logging.getLogger(__name__) -router = APIRouter(prefix="/user/cas", tags=["cas"]) - - -@router.get("/config") -async def config(): - return JSONResponse( - status_code=HTTPStatus.OK, - content={"message": "success", "data": get_cas_config()}, - ) - - -@router.get("/login") -async def login(redirect: str = Query("/", description="URL to return to after login")): - try: - login_url = _require_cas_server_redirect(build_login_url(redirect)) - return RedirectResponse(url=login_url, status_code=HTTPStatus.FOUND) - except CasAuthenticationError as exc: - logger.warning("CAS login rejected: %s", exc) - raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail="CAS login is not available") - - -@router.get("/callback") -async def callback(ticket: str = "", redirect: str = "/"): - try: - result = await login_with_ticket(ticket, redirect) - return JSONResponse( - status_code=HTTPStatus.OK, - content={"message": "CAS login successful", "data": result}, - ) - except CasAuthenticationError as exc: - logger.warning("CAS callback rejected: %s", exc) - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail="CAS authentication failed") - except Exception as exc: - logger.error(f"CAS callback failed: {exc}") - raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail="CAS login failed") - - -@router.post("/callback") -async def callback_logout(request: Request, logout_request: Optional[str] = None): - return await _handle_logout_request(request, logout_request, endpoint="callback") - - -@router.get("/renew") -async def renew(): - try: - return RedirectResponse(url=build_renew_url(), status_code=HTTPStatus.FOUND) - except CasAuthenticationError as exc: - logger.warning("CAS renew rejected: %s", exc) - return _renew_html(False, "CAS renew failed") - - -@router.get("/renew_callback") -async def renew_callback(ticket: str = ""): - if not ticket: - return _renew_html(False, "CAS session is not active") - try: - result = await renew_with_ticket(ticket) - return JSONResponse( - status_code=HTTPStatus.OK, - content={"message": "CAS renew successful", "data": result}, - ) - except Exception as exc: - logger.warning(f"CAS renew failed: {exc}") - return _renew_html(False, "CAS renew failed") - - -@router.post("/logout_callback") -async def logout_callback( - request: Request, - logout_request: Optional[str] = None, -): - return await _handle_logout_request(request, logout_request, endpoint="logout_callback") - - -async def _handle_logout_request( - request: Request, - logout_request: Optional[str] = None, - endpoint: str = "unknown", -): - logout_request = await _extract_logout_request(request, logout_request) - logger.info( - "CAS SLO %s received logoutRequest: present=%s length=%s", - endpoint, - bool(logout_request), - len(logout_request or ""), - ) - result = revoke_from_logout_request(logout_request) - logger.info("CAS SLO %s revoke result: %s", endpoint, result) - return JSONResponse( - status_code=HTTPStatus.OK, - content={"message": "success", "data": result}, - ) - - -async def _extract_logout_request(request: Request, logout_request: Optional[str] = None) -> str: - if logout_request: - return logout_request - - query_logout_request = request.query_params.get("logoutRequest") or request.query_params.get("logout_request") - if query_logout_request: - return query_logout_request - - body = await request.body() - raw_body = body.decode("utf-8") if body else "" - if not raw_body: - return "" - - parsed = parse_qs(raw_body) - return (parsed.get("logoutRequest") or parsed.get("logout_request") or [raw_body])[0] - - -def _renew_html(success: bool, reason: str = "") -> HTMLResponse: - status = "success" if success else "failed" - safe_reason = html.escape(reason) - return HTMLResponse( - status_code=HTTPStatus.OK, - content=f""" -""", - ) - - -def _require_cas_server_redirect(url: str) -> str: - parsed_url = urlsplit(url) - parsed_cas = urlsplit(CAS_SERVER_URL) - if ( - parsed_url.scheme not in {"http", "https"} - or not parsed_url.netloc - or parsed_url.scheme != parsed_cas.scheme - or parsed_url.netloc != parsed_cas.netloc - ): - logger.warning("Blocked CAS redirect outside configured server: %s", url) - raise CasAuthenticationError("Invalid CAS redirect URL") - return url diff --git a/backend/apps/config_app.py b/backend/apps/config_app.py index a818ec7cb..8cb383df7 100644 --- a/backend/apps/config_app.py +++ b/backend/apps/config_app.py @@ -2,7 +2,6 @@ from apps.app_factory import create_app from apps.agent_app import agent_config_router as agent_router -from apps.agent_repository_app import agent_repository_router from apps.config_sync_app import router as config_sync_router from apps.datamate_app import router as datamate_router from apps.vectordatabase_app import router as vectordatabase_router @@ -33,7 +32,6 @@ from apps.monitoring_app import router as monitoring_router from apps.a2a_server_app import router as a2a_server_router from apps.haotian_app import router as haotian_router -from apps.cas_app import router as cas_router from consts.const import IS_SPEED_MODE from services.prompt_template_service import sync_system_default_prompt_template @@ -56,7 +54,6 @@ async def sync_default_prompt_template_on_startup(): app.include_router(model_manager_router) app.include_router(config_sync_router) app.include_router(agent_router) -app.include_router(agent_repository_router) app.include_router(vectordatabase_router) app.include_router(datamate_router) app.include_router(voice_router) @@ -76,7 +73,6 @@ async def sync_default_prompt_template_on_startup(): app.include_router(user_management_router) app.include_router(oauth_router) -app.include_router(cas_router) app.include_router(summary_router) app.include_router(prompt_router) diff --git a/backend/apps/northbound_app.py b/backend/apps/northbound_app.py index 9f3b7e323..e6aff8e06 100644 --- a/backend/apps/northbound_app.py +++ b/backend/apps/northbound_app.py @@ -1,16 +1,14 @@ import logging from http import HTTPStatus from typing import Optional, Dict, Any -from urllib.parse import urlparse, unquote -import re +from urllib.parse import urlparse import uuid import httpx -from fastapi import APIRouter, Body, File, Header, HTTPException, Query, Request, UploadFile +from fastapi import APIRouter, Body, Header, Request, HTTPException, Query from fastapi.responses import JSONResponse, StreamingResponse -from consts.exceptions import LimitExceededError, UnauthorizedError, ConversationNotFoundError -from consts.model import ToolParamsRequest +from consts.exceptions import LimitExceededError, UnauthorizedError from services.northbound_service import ( NorthboundContext, get_conversation_history, @@ -19,35 +17,16 @@ stop_chat, get_agent_info_list, update_conversation_title, - upload_files_for_northbound, ) from utils.auth_utils import validate_bearer_token, get_user_and_tenant_by_access_key -from .file_management_app import build_content_disposition_header - router = APIRouter(prefix="/nb/v1", tags=["northbound"]) __all__ = ["router", "_get_northbound_context"] -def _resolve_proxy_download_filename(presigned_url: str, content_disposition: str) -> str: - """Resolve a stable download filename for the northbound file proxy.""" - if content_disposition: - filename_star_match = re.search(r"filename\*=UTF-8''([^;]+)", content_disposition) - if filename_star_match: - return unquote(filename_star_match.group(1)) or "download" - - filename_match = re.search(r'filename="?([^";]+)"?', content_disposition) - if filename_match: - return filename_match.group(1) or "download" - - path = unquote(urlparse(presigned_url).path) - filename = path.split("/")[-1].strip() - return filename or "download" - - async def _get_northbound_context(request: Request) -> NorthboundContext: """ Build northbound context from request. @@ -130,119 +109,13 @@ async def health_check(): return {"status": "healthy", "service": "northbound-api"} -@router.post( - "/chat/attachments/upload", - summary="Upload chat attachments for northbound runs", - description=( - "Upload one or more files for later use in `/nb/v1/chat/run`. " - "Successful uploads return reusable `s3_url` references." - ), -) -async def upload_chat_attachments( - request: Request, - files: list[UploadFile] = File( - ..., - description="List of files to upload", - examples=["report.pdf", "diagram.png"], - ), -): - try: - ctx: NorthboundContext = await _get_northbound_context(request) - return JSONResponse( - status_code=HTTPStatus.OK, - content=await upload_files_for_northbound(ctx=ctx, files=files), - ) - except LimitExceededError as e: - logging.error(f"Too Many Requests: rate limit exceeded: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.TOO_MANY_REQUESTS, - detail="Too Many Requests: rate limit exceeded") - except ValueError as e: - logging.error(f"Invalid northbound upload request: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(e)) - except PermissionError as e: - logging.error(f"Permission denied while uploading northbound files: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.FORBIDDEN, detail=str(e)) - except HTTPException as e: - raise e - except Exception as e: - logging.error(f"Failed to upload northbound files: {str(e)}", exc_info=e) - raise HTTPException( - status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail="Internal Server Error") - - -@router.post( - "/chat/run", - summary="Start a northbound chat run with optional attachments", - description=( - "Run a northbound chat request. Upload attachments first through " - "`/nb/v1/chat/attachments/upload`, then pass the returned `s3_url` values " - "through the `attachments` field." - ), -) +@router.post("/chat/run") async def run_chat( request: Request, - conversation_id: Optional[int] = Body( - None, - embed=True, - description="Existing conversation ID. Omit to create a new conversation.", - examples=[123], - ), - agent_name: str = Body( - ..., - embed=True, - description="Target agent name.", - examples=["general-assistant"], - ), - query: str = Body( - ..., - embed=True, - description="User input to send to the agent.", - examples=["Summarize the uploaded report and list the key risks."], - ), - attachments: Optional[list] = Body( - None, - embed=True, - description="Attachments for the chat. Can be either a list of S3 URL strings" - "or a list of attachment objects with full metadata.", - examples=[["s3://nexent/attachments/user123/20260609_report.pdf"]], - ), - meta_data: Optional[Dict[str, Any]] = Body( - None, - embed=True, - description="Optional metadata passed through for audit and usage logging.", - examples=[{"source": "crm", "ticket_id": "INC-1001"}], - ), - tool_params: Optional[ToolParamsRequest] = Body( - None, - embed=True, - description="Optional request-scoped overrides for tool initialization parameters. " - "Overrides DB-persisted params (ag_tool_instance_t.params) on a per-run basis. " - "Conflict resolution: request value wins over DB value. " - "Structure: agents -> {agent_name} -> tools -> {tool_name} -> {param_name: param_value}. " - "tool_name matching: first by tool.name, then by tool.class_name. " - "Unknown param names cause a ValidationError (400). " - "Metadata-derived fields (e.g., vdb_core, embedding_model) are recalculated " - "from merged params for tools like KnowledgeBaseSearchTool, DifySearchTool, DataMateSearchTool.", - examples=[{ - "agents": { - "common_sense_qa_assistant": { - "tools": { - "analyze_text_file": { - "chunk_size": 4000, - "summary_only": True, - "prompt": "Please provide a concise summary of this document focusing on key facts." - }, - "knowledge_base_search": { - "top_k": 10, - "rerank": True, - "rerank_model_name": "gte-rerank-v2", - "index_names": ["nexent-docs", "faq-index"] - } - } - } - } - }], - ), + conversation_id: Optional[int] = Body(None, embed=True), + agent_name: str = Body(..., embed=True), + query: str = Body(..., embed=True), + meta_data: Optional[Dict[str, Any]] = Body(None, embed=True), idempotency_key: Optional[str] = Header(None, alias="Idempotency-Key"), ): try: @@ -252,21 +125,13 @@ async def run_chat( conversation_id=conversation_id, agent_name=agent_name, query=query, - attachments=attachments, meta_data=meta_data, - tool_params=tool_params, idempotency_key=idempotency_key, ) except LimitExceededError as e: logging.error(f"Too Many Requests: rate limit exceeded: {str(e)}", exc_info=e) raise HTTPException(status_code=HTTPStatus.TOO_MANY_REQUESTS, detail="Too Many Requests: rate limit exceeded") - except ValueError as e: - logging.error(f"Invalid northbound chat request: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(e)) - except PermissionError as e: - logging.error(f"Permission denied while running northbound chat: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.FORBIDDEN, detail=str(e)) except HTTPException as e: raise e except Exception as e: @@ -389,9 +254,6 @@ async def update_convs_title( logging.error(f"Too Many Requests: rate limit exceeded: {str(e)}", exc_info=e) raise HTTPException(status_code=HTTPStatus.TOO_MANY_REQUESTS, detail="Too Many Requests: rate limit exceeded") - except ConversationNotFoundError as e: - logging.error(f"Conversation not found while updating title: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(e)) except HTTPException as e: raise e except Exception as e: @@ -450,12 +312,12 @@ async def fetch_file_from_presigned_url( content_type = response.headers.get("Content-Type", "application/octet-stream") content_disposition = response.headers.get("Content-Disposition", "") - download_filename = _resolve_proxy_download_filename(presigned_url, content_disposition) headers = { "Content-Type": content_type, - "Content-Disposition": build_content_disposition_header(download_filename), } + if content_disposition: + headers["Content-Disposition"] = content_disposition return StreamingResponse( content=response.aiter_bytes(), diff --git a/backend/apps/northbound_knowledge_app.py b/backend/apps/northbound_knowledge_app.py index 02739d138..775d6c567 100644 --- a/backend/apps/northbound_knowledge_app.py +++ b/backend/apps/northbound_knowledge_app.py @@ -51,8 +51,7 @@ async def _require_asset_owner_context(request: Request) -> NorthboundContext: @router.get("/indices") async def get_list_indices( request: Request, - pattern: Annotated[str, Query( - description="Pattern to match index names")] = "*", + pattern: Annotated[str, Query(description="Pattern to match index names")] = "*", ): """List knowledge bases visible to the asset-owner tenant. @@ -93,7 +92,7 @@ async def create_new_index( Optional[Dict[str, Any]], Body( description=( - "Request body with optional fields (ingroup_permission, group_ids, embedding_model_name, preserve_source_file)" + "Request body with optional fields (ingroup_permission, group_ids, embedding_model_name)" ), ), ] = None, @@ -111,12 +110,10 @@ async def create_new_index( ingroup_permission = None group_ids = None embedding_model_name = None - preserve_source_file = None if body: ingroup_permission = body.get("ingroup_permission") group_ids = body.get("group_ids") embedding_model_name = body.get("embedding_model_name") - preserve_source_file = body.get("preserve_source_file") return ElasticSearchService.create_knowledge_base( knowledge_name=index_name, @@ -127,7 +124,6 @@ async def create_new_index( ingroup_permission=ingroup_permission, group_ids=group_ids, embedding_model_name=embedding_model_name, - preserve_source_file=preserve_source_file, ) except LimitExceededError as e: logger.exception("Rate limit exceeded while creating index") @@ -226,65 +222,52 @@ async def delete_documents( request: Request, index_name: Annotated[str, Path(..., description="Name of the index")], path_or_url: Annotated[str, Query(..., description="Path or URL of documents to delete")], - scope: Annotated[ - str, - Query( - description=( - "source_only: delete MinIO source only; " - "full: delete ES, MinIO, and Redis records" - ), - ), - ] = "full", ): - """Delete a document by scope. Restricted to asset administrators.""" + """Delete documents by path or URL and clean up related Redis records. + + Restricted to asset administrators (same auth as get_list_indices). + """ try: - await _require_asset_owner_context(request) + ctx = await _require_asset_owner_context(request) vdb_core = get_vector_db_core(db_type=VectorDatabaseType.ELASTICSEARCH) - logger.debug( - "Deleting documents for index %s scope=%s", index_name, scope - ) - result = await ElasticSearchService.delete_document_by_scope( - index_name, path_or_url, scope, vdb_core - ) + logger.debug("Deleting documents for index %s", index_name) + result = ElasticSearchService.delete_documents( + index_name, path_or_url, vdb_core) + + try: + redis_service = get_redis_service() + redis_cleanup_result = redis_service.delete_document_records( + index_name, path_or_url) + + result["redis_cleanup"] = redis_cleanup_result + + original_message = result.get( + "message", "Documents deleted successfully") + result["message"] = ( + f"{original_message}. " + f"Cleaned up {redis_cleanup_result['total_deleted']} Redis records " + f"({redis_cleanup_result['celery_tasks_deleted']} tasks, " + f"{redis_cleanup_result['cache_keys_deleted']} cache keys)." + ) - if scope == "full": - try: - redis_service = get_redis_service() - redis_cleanup_result = redis_service.delete_document_records( - index_name, path_or_url - ) - result["redis_cleanup"] = redis_cleanup_result - original_message = result.get( - "message", "Documents deleted successfully" - ) - result["message"] = ( - f"{original_message}. " - f"Cleaned up {redis_cleanup_result['total_deleted']} Redis records " - f"({redis_cleanup_result['celery_tasks_deleted']} tasks, " - f"{redis_cleanup_result['cache_keys_deleted']} cache keys)." - ) - if redis_cleanup_result.get("errors"): - result["redis_warnings"] = redis_cleanup_result["errors"] - except Exception as redis_error: - logger.warning( - "Redis cleanup failed for index %s: %s", - index_name, - redis_error, - ) - result["redis_cleanup_error"] = str(redis_error) - original_message = result.get( - "message", "Documents deleted successfully" - ) - result["message"] = ( - f"{original_message}, but Redis cleanup encountered an error: " - f"{str(redis_error)}" - ) + if redis_cleanup_result.get("errors"): + result["redis_warnings"] = redis_cleanup_result["errors"] + + except Exception as redis_error: + logger.warning( + "Redis cleanup failed for index %s: %s", + index_name, + redis_error, + ) + result["redis_cleanup_error"] = str(redis_error) + original_message = result.get( + "message", "Documents deleted successfully") + result["message"] = ( + f"{original_message}, but Redis cleanup encountered an error: " + f"{str(redis_error)}" + ) return result - except ValueError as exc: - raise HTTPException( - status_code=HTTPStatus.BAD_REQUEST, detail=str(exc) - ) except LimitExceededError as e: logger.exception("Rate limit exceeded while deleting documents") raise HTTPException( diff --git a/backend/apps/prompt_app.py b/backend/apps/prompt_app.py index 6b82a5c82..987729e69 100644 --- a/backend/apps/prompt_app.py +++ b/backend/apps/prompt_app.py @@ -4,19 +4,11 @@ from fastapi import APIRouter, Header, Request from fastapi.responses import JSONResponse, StreamingResponse -from consts.model import ( - GeneratePromptRequest, - OptimizePromptSectionRequest, - OptimizePromptBadCaseRequest, - OptimizePromptFromDebugRequest, -) +from consts.model import GeneratePromptRequest, OptimizePromptSectionRequest from services.prompt_service import ( gen_system_prompt_streamable, - OptimizeRequest, - OptimizeResult, - PromptOptimizationService, + optimize_prompt_section_impl, ) -from adapters.exception import NexentCapabilityError from utils.auth_utils import get_current_user_info router = APIRouter(prefix="/prompt") @@ -56,140 +48,30 @@ async def optimize_prompt_section_api( http_request: Request, authorization: Optional[str] = Header(None) ): - _, tenant_id, language = get_current_user_info( - authorization, http_request) - - service = PromptOptimizationService( - model_id=optimize_request.model_id, - tenant_id=tenant_id, - language=language, - ) - - try: - result = service.optimize( - OptimizeRequest( - agent_id=optimize_request.agent_id, - model_id=optimize_request.model_id, - task_description=optimize_request.task_description, - section_type=optimize_request.section_type, - section_title=optimize_request.section_title, - current_content=optimize_request.current_content, - feedback=optimize_request.feedback, - mode=optimize_request.mode, - start_pos=optimize_request.start_pos, - end_pos=optimize_request.end_pos, - tool_ids=optimize_request.tool_ids, - sub_agent_ids=optimize_request.sub_agent_ids, - knowledge_base_display_names=optimize_request.knowledge_base_display_names, - ) - ) - return JSONResponse( - status_code=HTTPStatus.OK, - content={ - "message": "Success", - "data": { - "optimized_content": result.optimized_content, - "section_type": result.section_type, - "section_title": result.section_title, - "original_content": result.original_content, - } - }, - headers={"X-Prompt-Source": result.source}, - ) - except NexentCapabilityError as e: - return JSONResponse( - status_code=HTTPStatus.BAD_REQUEST, - content={"message": str(e)}, - ) - except Exception as exc: - logger.exception(f"Error occurred while optimizing prompt section: {exc}") - raise - - -@router.post("/optimize/badcase") -async def optimize_prompt_badcase_api( - badcase_request: OptimizePromptBadCaseRequest, - http_request: Request, - authorization: Optional[str] = Header(None) -): - _, tenant_id, language = get_current_user_info( - authorization, http_request) - - service = PromptOptimizationService( - model_id=badcase_request.model_id, - tenant_id=tenant_id, - language=language, - ) - - try: - result = service.optimize_badcase( - current_content=badcase_request.current_content, - bad_cases=badcase_request.bad_cases, - agent_id=badcase_request.agent_id, - section_type=badcase_request.section_type, - section_title=badcase_request.section_title, - tool_ids=badcase_request.tool_ids, - sub_agent_ids=badcase_request.sub_agent_ids, - knowledge_base_display_names=badcase_request.knowledge_base_display_names, - ) - return JSONResponse( - status_code=HTTPStatus.OK, - content={ - "message": "Success", - "data": { - "optimized_content": result.optimized_content, - "section_type": result.section_type, - "section_title": result.section_title, - "original_content": result.original_content, - } - }, - headers={"X-Prompt-Source": result.source}, - ) - except NexentCapabilityError as e: - return JSONResponse( - status_code=HTTPStatus.BAD_REQUEST, - content={"message": str(e)}, - ) - - -@router.post("/optimize/from_debug") -async def optimize_prompt_from_debug_api( - optimize_request: OptimizePromptFromDebugRequest, - http_request: Request, - authorization: Optional[str] = Header(None) -): - _, tenant_id, language = get_current_user_info( - authorization, http_request) - - service = PromptOptimizationService( - model_id=optimize_request.model_id, - tenant_id=tenant_id, - language=language, - ) - try: - result = service.optimize_from_debug( + _, tenant_id, language = get_current_user_info( + authorization, http_request) + optimized_section = optimize_prompt_section_impl( agent_id=optimize_request.agent_id, + model_id=optimize_request.model_id, + task_description=optimize_request.task_description, + tenant_id=tenant_id, + language=language, + section_type=optimize_request.section_type, + section_title=optimize_request.section_title, + current_content=optimize_request.current_content, feedback=optimize_request.feedback, - selected=optimize_request.selected, - history=optimize_request.history, + tool_ids=optimize_request.tool_ids, + sub_agent_ids=optimize_request.sub_agent_ids, + knowledge_base_display_names=optimize_request.knowledge_base_display_names, ) return JSONResponse( status_code=HTTPStatus.OK, content={ - "message": "Success", - "data": { - "original_full_prompt": result.original_content, - "optimized_full_prompt": result.optimized_content, - } - }, - headers={"X-Prompt-Source": result.source}, - ) - except NexentCapabilityError as e: - return JSONResponse( - status_code=HTTPStatus.BAD_REQUEST, - content={"message": str(e)}, + "message": "Prompt section optimized successfully", + "data": optimized_section, + } ) except Exception as exc: - logger.exception(f"Error occurred while optimizing prompt from debug: {exc}") + logger.exception(f"Error occurred while optimizing prompt section: {exc}") raise diff --git a/backend/apps/tool_config_app.py b/backend/apps/tool_config_app.py index bfc8d5ca0..f0b7f9304 100644 --- a/backend/apps/tool_config_app.py +++ b/backend/apps/tool_config_app.py @@ -160,14 +160,12 @@ async def import_openapi_service_api( server_url: Base URL of the REST API server openapi_json: Complete OpenAPI JSON specification service_description: Optional service description - headers_template: Optional default headers template force_update: If True, replace all existing tools for this service """ service_name = openapi_service_request.get("service_name") server_url = openapi_service_request.get("server_url") openapi_json = openapi_service_request.get("openapi_json") service_description = openapi_service_request.get("service_description") - headers_template = openapi_service_request.get("headers_template") force_update = openapi_service_request.get("force_update", False) if not service_name: @@ -194,7 +192,6 @@ async def import_openapi_service_api( tenant_id=tenant_id, user_id=user_id, service_description=service_description, - headers_template=headers_template, force_update=force_update ) diff --git a/backend/apps/user_management_app.py b/backend/apps/user_management_app.py index e79fde887..edbcdf27d 100644 --- a/backend/apps/user_management_app.py +++ b/backend/apps/user_management_app.py @@ -19,13 +19,12 @@ ValidationError, ) from consts.error_code import ErrorCode -from services.cas_service import build_logout_url, CasAuthenticationError from services.user_management_service import get_authorized_client, validate_token, \ check_auth_service_health, signup_user_with_invitation, signin_user, refresh_user_token, \ get_session_by_authorization, get_user_info, create_token, list_tokens_by_user, delete_token, \ update_password from services.user_service import delete_user_and_cleanup -from utils.auth_utils import get_current_user_id, extract_session_id_from_authorization +from utils.auth_utils import get_current_user_id load_dotenv() @@ -151,18 +150,7 @@ async def logout(request: Request): authorization = request.headers.get("Authorization") try: # Make logout idempotent: if no token or token expired, still return success - session_id = None - cas_logout_url = "" if authorization: - session_id = extract_session_id_from_authorization(authorization) - if session_id: - from database.cas_session_db import revoke_cas_session_by_session_id - - revoke_cas_session_by_session_id(session_id, actor="user") - try: - cas_logout_url = build_logout_url() - except CasAuthenticationError as cas_err: - logging.warning(f"CAS logout URL is unavailable: {str(cas_err)}") client = get_authorized_client(authorization) try: client.auth.sign_out() @@ -171,12 +159,7 @@ async def logout(request: Request): logging.warning( f"Sign out encountered an error but will be ignored: {str(signout_err)}") return JSONResponse(status_code=HTTPStatus.OK, - content={ - "message": "Logout successful", - "data": { - "cas_logout_url": cas_logout_url - } - }) + content={"message": "Logout successful"}) except Exception as e: logging.error(f"User logout failed: {str(e)}") @@ -231,10 +214,6 @@ async def get_user_information(request: Request): if not user_info: raise UnauthorizedError("User information not found") - user_info["user"]["auth_provider"] = ( - "cas" if extract_session_id_from_authorization(authorization) else "local" - ) - return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": user_info}) diff --git a/backend/apps/vectordatabase_app.py b/backend/apps/vectordatabase_app.py index 505c39559..118537766 100644 --- a/backend/apps/vectordatabase_app.py +++ b/backend/apps/vectordatabase_app.py @@ -76,7 +76,7 @@ def create_new_index( embedding_dim: Optional[int] = Query( None, description="Dimension of the embedding vectors"), request: Dict[str, Any] = Body( - None, description="Request body with optional fields (ingroup_permission, group_ids, embedding_model_name, preserve_source_file)"), + None, description="Request body with optional fields (ingroup_permission, group_ids, embedding_model_name)"), vdb_core: VectorDatabaseCore = Depends(get_vector_db_core), authorization: Optional[str] = Header(None) ): @@ -89,13 +89,11 @@ def create_new_index( group_ids = None embedding_model_name: Optional[str] = None is_multimodal: Optional[bool] = None - preserve_source_file: Optional[bool] = None if request: ingroup_permission = request.get("ingroup_permission") group_ids = request.get("group_ids") embedding_model_name = request.get("embeddingModel") is_multimodal = request.get("is_multimodal") - preserve_source_file = request.get("preserve_source_file") # Treat path parameter as user-facing knowledge base name for new creations return ElasticSearchService.create_knowledge_base( @@ -108,7 +106,6 @@ def create_new_index( group_ids=group_ids, embedding_model_name=embedding_model_name, is_multimodal=is_multimodal, - preserve_source_file=preserve_source_file, ) except Exception as e: raise HTTPException( @@ -508,70 +505,54 @@ async def get_index_files( @router.delete("/{index_name}/documents") -async def delete_documents( +def delete_documents( index_name: str = Path(..., description="Name of the index"), path_or_url: str = Query(..., description="Path or URL of documents to delete"), - scope: str = Query( - "full", - description=( - "source_only: delete MinIO source only, keep ES chunks/vectors; " - "full: delete ES documents, MinIO source, and Redis task records" - ), - ), vdb_core: VectorDatabaseCore = Depends(get_vector_db_core) ): - """Delete a document by scope: source file only or full removal from the index.""" + """Delete documents by path or URL and clean up related Redis records""" try: - result = await ElasticSearchService.delete_document_by_scope( - index_name, path_or_url, scope, vdb_core - ) + # First delete the documents using existing service + result = ElasticSearchService.delete_documents( + index_name, path_or_url, vdb_core) + + # Then clean up Redis records related to this specific document + try: + redis_service = get_redis_service() + redis_cleanup_result = redis_service.delete_document_records( + index_name, path_or_url) + + # Add Redis cleanup info to the result + result["redis_cleanup"] = redis_cleanup_result + + # Update the message to include Redis cleanup info + original_message = result.get( + "message", "Documents deleted successfully") + result["message"] = ( + f"{original_message}. " + f"Cleaned up {redis_cleanup_result['total_deleted']} Redis records " + f"({redis_cleanup_result['celery_tasks_deleted']} tasks, " + f"{redis_cleanup_result['cache_keys_deleted']} cache keys)." + ) - if scope == "full": - try: - redis_service = get_redis_service() - redis_cleanup_result = redis_service.delete_document_records( - index_name, path_or_url - ) - result["redis_cleanup"] = redis_cleanup_result - original_message = result.get( - "message", "Documents deleted successfully" - ) - result["message"] = ( - f"{original_message}. " - f"Cleaned up {redis_cleanup_result['total_deleted']} Redis records " - f"({redis_cleanup_result['celery_tasks_deleted']} tasks, " - f"{redis_cleanup_result['cache_keys_deleted']} cache keys)." - ) - if redis_cleanup_result.get("errors"): - result["redis_warnings"] = redis_cleanup_result["errors"] - except Exception as redis_error: - logger.warning( - "Redis cleanup failed for document %s in index %s: %s", - path_or_url, - index_name, - redis_error, - ) - result["redis_cleanup_error"] = str(redis_error) - original_message = result.get( - "message", "Documents deleted successfully" - ) - result["message"] = ( - f"{original_message}, but Redis cleanup encountered an error: " - f"{str(redis_error)}" - ) + if redis_cleanup_result.get("errors"): + result["redis_warnings"] = redis_cleanup_result["errors"] + + except Exception as redis_error: + logger.warning( + f"Redis cleanup failed for document {path_or_url} in index {index_name}: {str(redis_error)}") + result["redis_cleanup_error"] = str(redis_error) + original_message = result.get( + "message", "Documents deleted successfully") + result[ + "message"] = f"{original_message}, but Redis cleanup encountered an error: {str(redis_error)}" return result - except ValueError as exc: - raise HTTPException( - status_code=HTTPStatus.BAD_REQUEST, detail=str(exc) - ) except Exception as e: raise HTTPException( - status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail=f"Error delete indexing documents: {e}", - ) + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail=f"Error delete indexing documents: {e}") @router.get("/{index_name}/documents/{path_or_url:path}/error-info") diff --git a/backend/consts/const.py b/backend/consts/const.py index 574d550c0..ac2196c2a 100644 --- a/backend/consts/const.py +++ b/backend/consts/const.py @@ -90,31 +90,6 @@ class VectorDatabaseType(str, Enum): OAUTH_CA_BUNDLE = os.getenv("OAUTH_CA_BUNDLE", "") -# CAS SSO Configuration -CAS_ENABLED = os.getenv("CAS_ENABLED", "false").lower() in ("true", "1", "yes", "on") -CAS_SERVER_URL = os.getenv("CAS_SERVER_URL", "").rstrip("/") -CAS_VALIDATE_PATH = os.getenv("CAS_VALIDATE_PATH", "/p3/serviceValidate") -CAS_CALLBACK_BASE_URL = os.getenv("CAS_CALLBACK_BASE_URL", OAUTH_CALLBACK_BASE_URL).rstrip("/") -# CAS login mode: -# - disabled: disable CAS login entry and automatic CAS redirects. -# - button: show CAS as an optional login entry. -# - force: automatically redirect unauthenticated users to CAS login. -CAS_LOGIN_MODE = os.getenv("CAS_LOGIN_MODE", "disabled").lower() -CAS_USER_ATTRIBUTE = os.getenv("CAS_USER_ATTRIBUTE", "") -CAS_EMAIL_ATTRIBUTE = os.getenv("CAS_EMAIL_ATTRIBUTE", "email") -CAS_ROLE_ATTRIBUTE = os.getenv("CAS_ROLE_ATTRIBUTE", "role") -CAS_TENANT_ATTRIBUTE = os.getenv("CAS_TENANT_ATTRIBUTE", "tenant_id") -CAS_ROLE_MAP_JSON = os.getenv("CAS_ROLE_MAP_JSON", "") -CAS_SESSION_MAX_AGE_SECONDS = int(os.getenv("CAS_SESSION_MAX_AGE_SECONDS", "3600") or 3600) -LOCAL_SESSION_MAX_AGE_SECONDS = int(os.getenv("LOCAL_SESSION_MAX_AGE_SECONDS", "3600") or 3600) -CAS_RENEW_BEFORE_SECONDS = int(os.getenv("CAS_RENEW_BEFORE_SECONDS", "300") or 300) -CAS_RENEW_TIMEOUT_SECONDS = int(os.getenv("CAS_RENEW_TIMEOUT_SECONDS", "10") or 10) -CAS_SYNTHETIC_EMAIL_DOMAIN = os.getenv("CAS_SYNTHETIC_EMAIL_DOMAIN", "cas.local") -CAS_LOGOUT_URL = os.getenv("CAS_LOGOUT_URL", "") -CAS_SSL_VERIFY = os.getenv("CAS_SSL_VERIFY", "true").lower() == "true" -CAS_CA_BUNDLE = os.getenv("CAS_CA_BUNDLE", "") - - # ===== To be migrated to frontend configuration ===== # Email Configuration IMAP_SERVER = os.getenv('IMAP_SERVER') @@ -233,7 +208,6 @@ class VectorDatabaseType(str, Enum): "NEXENT_MCP_DOCKER_IMAGE", "nexent/nexent-mcp:latest") ENABLE_UPLOAD_IMAGE = os.getenv( "ENABLE_UPLOAD_IMAGE", "false").lower() == "true" -ENABLE_JIUWEN_SDK = os.getenv("NEXENT_ENABLE_JIUWEN_SDK", "true").lower() == "true" # Celery Configuration @@ -401,47 +375,36 @@ class VectorDatabaseType(str, Enum): OTEL_SERVICE_NAME = OTEL_SERVICE_NAME_RAW or "nexent-backend" OTEL_EXPORTER_OTLP_ENDPOINT_RAW = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") OTEL_EXPORTER_OTLP_ENDPOINT = OTEL_EXPORTER_OTLP_ENDPOINT_RAW or "http://localhost:4318" -OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = os.getenv( - "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") -OTEL_EXPORTER_OTLP_METRICS_ENDPOINT = os.getenv( - "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "") +OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = os.getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") +OTEL_EXPORTER_OTLP_METRICS_ENDPOINT = os.getenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "") OTEL_EXPORTER_OTLP_PROTOCOL_RAW = os.getenv("OTEL_EXPORTER_OTLP_PROTOCOL") OTEL_EXPORTER_OTLP_PROTOCOL = OTEL_EXPORTER_OTLP_PROTOCOL_RAW or "http" OTEL_EXPORTER_OTLP_HEADERS_RAW = os.getenv("OTEL_EXPORTER_OTLP_HEADERS") OTEL_EXPORTER_OTLP_HEADERS = OTEL_EXPORTER_OTLP_HEADERS_RAW or "" -OTEL_EXPORTER_OTLP_AUTHORIZATION = os.getenv( - "OTEL_EXPORTER_OTLP_AUTHORIZATION", "") +OTEL_EXPORTER_OTLP_AUTHORIZATION = os.getenv("OTEL_EXPORTER_OTLP_AUTHORIZATION", "") OTEL_EXPORTER_OTLP_X_API_KEY = os.getenv("OTEL_EXPORTER_OTLP_X_API_KEY", "") OTEL_EXPORTER_OTLP_LANGFUSE_INGESTION_VERSION = os.getenv( "OTEL_EXPORTER_OTLP_LANGFUSE_INGESTION_VERSION", "") LANGSMITH_API_KEY = os.getenv("LANGSMITH_API_KEY", "") LANGSMITH_PROJECT = os.getenv("LANGSMITH_PROJECT", "") -OTEL_EXPORTER_OTLP_METRICS_ENABLED_RAW = os.getenv( - "OTEL_EXPORTER_OTLP_METRICS_ENABLED") +OTEL_EXPORTER_OTLP_METRICS_ENABLED_RAW = os.getenv("OTEL_EXPORTER_OTLP_METRICS_ENABLED") OTEL_EXPORTER_OTLP_METRICS_ENABLED = ( OTEL_EXPORTER_OTLP_METRICS_ENABLED_RAW or "true").lower() == "true" -MONITORING_INSTRUMENT_REQUESTS_RAW = os.getenv( - "MONITORING_INSTRUMENT_REQUESTS") +MONITORING_INSTRUMENT_REQUESTS_RAW = os.getenv("MONITORING_INSTRUMENT_REQUESTS") MONITORING_INSTRUMENT_REQUESTS = ( MONITORING_INSTRUMENT_REQUESTS_RAW or "false").lower() == "true" -MONITORING_FASTAPI_INCLUDED_URLS = os.getenv( - "MONITORING_FASTAPI_INCLUDED_URLS", "") -MONITORING_FASTAPI_EXCLUDED_URLS = os.getenv( - "MONITORING_FASTAPI_EXCLUDED_URLS", "") -MONITORING_FASTAPI_EXCLUDE_SPANS = os.getenv( - "MONITORING_FASTAPI_EXCLUDE_SPANS", "receive,send") +MONITORING_FASTAPI_INCLUDED_URLS = os.getenv("MONITORING_FASTAPI_INCLUDED_URLS", "") +MONITORING_FASTAPI_EXCLUDED_URLS = os.getenv("MONITORING_FASTAPI_EXCLUDED_URLS", "") +MONITORING_FASTAPI_EXCLUDE_SPANS = os.getenv("MONITORING_FASTAPI_EXCLUDE_SPANS", "receive,send") MONITORING_PROJECT_NAME = os.getenv("MONITORING_PROJECT_NAME", "") MONITORING_DASHBOARD_URL = os.getenv("MONITORING_DASHBOARD_URL", "") -MONITORING_TRACE_CONTENT_MODE = os.getenv( - "MONITORING_TRACE_CONTENT_MODE", "summary") +MONITORING_TRACE_CONTENT_MODE = os.getenv("MONITORING_TRACE_CONTENT_MODE", "summary") MONITORING_TRACE_MAX_CHARS = os.getenv("MONITORING_TRACE_MAX_CHARS", "4000") MONITORING_TRACE_MAX_ITEMS = os.getenv("MONITORING_TRACE_MAX_ITEMS", "20") TELEMETRY_SAMPLE_RATE_RAW = os.getenv("TELEMETRY_SAMPLE_RATE") TELEMETRY_SAMPLE_RATE = float(TELEMETRY_SAMPLE_RATE_RAW or "1.0") # Parse OTLP headers into dict format - - def _parse_otlp_headers(headers_str: str) -> dict: """Parse OTLP headers string into dict. Format: 'key1=value1,key2=value2'""" if not headers_str: @@ -453,7 +416,6 @@ def _parse_otlp_headers(headers_str: str) -> dict: headers[key.strip()] = value.strip() return headers - OTLP_HEADERS = _parse_otlp_headers(OTEL_EXPORTER_OTLP_HEADERS) if OTEL_EXPORTER_OTLP_AUTHORIZATION: OTLP_HEADERS["Authorization"] = OTEL_EXPORTER_OTLP_AUTHORIZATION @@ -486,7 +448,7 @@ def _parse_otlp_headers(headers_str: str) -> dict: # APP Version -APP_VERSION = "v2.2.1" +APP_VERSION = "v2.2.0" # Skill Creation Streaming Configuration diff --git a/backend/consts/model.py b/backend/consts/model.py index 00e5b8a0a..6969999fe 100644 --- a/backend/consts/model.py +++ b/backend/consts/model.py @@ -1,8 +1,8 @@ from enum import Enum -from typing import Optional, Any, List, Dict, Literal +from typing import Optional, Any, List, Dict from pydantic import BaseModel, Field, EmailStr, ConfigDict, field_validator -from nexent.core.agents.agent_model import AgentVerificationConfig, ToolConfig +from nexent.core.agents.agent_model import ToolConfig from consts.prompt_template import PROMPT_GENERATE_TEMPLATE_FIELD_ALIAS_MAP @@ -230,24 +230,6 @@ class HistoryItem(BaseModel): minio_files: Optional[List[Dict[str, Any]]] = None -class AgentToolParamsRequest(BaseModel): - """Request-scoped tool parameter overrides for a single agent.""" - - tools: Dict[str, Dict[str, Any]] = Field( - default_factory=dict, - description="Mapping from tool identifier to request-scoped override params", - ) - - -class ToolParamsRequest(BaseModel): - """Request-scoped tool parameter overrides for main and managed agents.""" - - agents: Dict[str, AgentToolParamsRequest] = Field( - default_factory=dict, - description="Mapping from agent identifier to tool parameter overrides", - ) - - class AgentRequest(BaseModel): query: str conversation_id: Optional[int] = None @@ -258,7 +240,6 @@ class AgentRequest(BaseModel): model_id: Optional[int] = None version_no: Optional[int] = None is_debug: Optional[bool] = False - tool_params: Optional[ToolParamsRequest] = None class MessageUnit(BaseModel): @@ -433,9 +414,6 @@ class OptimizePromptSectionRequest(BaseModel): section_title: str current_content: str feedback: str - mode: Literal["general", "insert", "select"] = "general" - start_pos: Optional[int] = Field(None, description="Start position for insert/select mode") - end_pos: Optional[int] = Field(None, description="End position for insert/select mode") tool_ids: Optional[List[int]] = Field( None, description="Optional: tool IDs from frontend (takes precedence over database query)") sub_agent_ids: Optional[List[int]] = Field( @@ -444,38 +422,6 @@ class OptimizePromptSectionRequest(BaseModel): None, description="Optional: knowledge base display names from frontend (takes precedence over database query)") -class BadCaseItem(BaseModel): - question: str - answer: str - label: Optional[str] = None - reason: Optional[str] = None - - -class OptimizePromptBadCaseRequest(BaseModel): - agent_id: int - model_id: int - current_content: str - bad_cases: List[BadCaseItem] - section_type: str - section_title: str - tool_ids: Optional[List[int]] = Field(None) - sub_agent_ids: Optional[List[int]] = Field(None) - knowledge_base_display_names: Optional[List[str]] = Field(None) - - -class OptimizeFromDebugSelected(BaseModel): - user_question: str - assistant_answer: str - - -class OptimizePromptFromDebugRequest(BaseModel): - agent_id: int - model_id: int - feedback: str - selected: OptimizeFromDebugSelected - history: Optional[List[HistoryItem]] = None - - class GenerateTitleRequest(BaseModel): conversation_id: int question: str @@ -508,18 +454,8 @@ class AgentInfoRequest(BaseModel): group_ids: Optional[List[int]] = None ingroup_permission: Optional[str] = None enable_context_manager: Optional[bool] = None - verification_config: Optional[Dict[str, Any]] = None - greeting_message: Optional[str] = None - example_questions: Optional[List[str]] = None version_no: int = 0 - @field_validator("verification_config", mode="before") - @classmethod - def normalize_verification_config(cls, value): - if value is None: - return None - return AgentVerificationConfig.model_validate(value).model_dump() - class AgentIDRequest(BaseModel): agent_id: int @@ -584,7 +520,6 @@ class MessageIdRequest(BaseModel): class ExportAndImportAgentInfo(BaseModel): agent_id: int - tenant_id: Optional[str] = None name: str display_name: Optional[str] = None description: str @@ -592,7 +527,6 @@ class ExportAndImportAgentInfo(BaseModel): author: Optional[str] = None max_steps: int provide_run_summary: bool - verification_config: Optional[Dict[str, Any]] = None duty_prompt: Optional[str] = None constraint_prompt: Optional[str] = None few_shots_prompt: Optional[str] = None @@ -622,11 +556,6 @@ class ExportAndImportDataFormat(BaseModel): mcp_info: List[MCPInfo] -class AgentRepositorySnapshot(ExportAndImportDataFormat): - """Frozen marketplace snapshot: export format plus optional skill ZIP payloads.""" - skills: Optional[List["SkillZipEntry"]] = None - - class SkillZipEntry(BaseModel): """A skill bundled inside an agent export ZIP.""" skill_name: str diff --git a/backend/data_process/tasks.py b/backend/data_process/tasks.py index 4dd6edd69..f2a30f9b7 100644 --- a/backend/data_process/tasks.py +++ b/backend/data_process/tasks.py @@ -8,11 +8,9 @@ import os import threading import time -from dataclasses import dataclass from typing import Any, Dict, Optional, List, Tuple import aiohttp -import requests import re import ray from celery import Task, chain, states, group, chord @@ -21,7 +19,6 @@ from utils.file_management_utils import get_file_size from database.attachment_db import get_file_stream -from database.knowledge_db import get_knowledge_record from services.redis_service import get_redis_service from .app import app from .ray_actors import DataProcessorRayActor @@ -46,12 +43,10 @@ logger = logging.getLogger("data_process.tasks") -ASYNC_SPLIT_RETRY_MAX = max( - FORWARD_REDIS_RETRY_MAX * 5, FORWARD_REDIS_RETRY_MAX) +ASYNC_SPLIT_RETRY_MAX = max(FORWARD_REDIS_RETRY_MAX * 5, FORWARD_REDIS_RETRY_MAX) FORWARD_ES_CHUNK_BATCH_SIZE = 64 IMAGE_METADATA_PROCESS_SOURCE = "UniversalImageExtractor" - def _wait_for_split_ready(redis_key: str, timeout_s: int, poll_interval_ms: int) -> int: """ Wait until async split aggregation is marked ready in Redis. @@ -96,8 +91,7 @@ def _estimate_parallel_parts() -> int: def _compute_split_wait_timeout(parts_count: int) -> int: base_timeout = DP_REDIS_CHUNKS_WAIT_TIMEOUT_S waves = math.ceil(max(1, parts_count) / _estimate_parallel_parts()) - dynamic_timeout = base_timeout + \ - max(0, waves - 1) * max(1, PER_WAVE_TIMEOUT) + dynamic_timeout = base_timeout + max(0, waves - 1) * max(1, PER_WAVE_TIMEOUT) return min(MAX_TIMEOUT, max(base_timeout, dynamic_timeout)) @@ -184,6 +178,7 @@ def _build_balanced_batches( return batches + # Thread lock for initializing Ray to prevent race conditions ray_init_lock = threading.Lock() @@ -332,35 +327,6 @@ def run_in_thread(): raise -def _delete_source_file_via_http_sync( - *, - base_url: str, - index_name: str, - path_or_url: str, - scope: str, - timeout_s: float = 30.0, -) -> Dict[str, Any]: - base = (base_url or "").rstrip("/") - if not base: - raise RuntimeError("ELASTICSEARCH_SERVICE is not configured") - url = f"{base}/indices/{index_name}/documents" - params = {"path_or_url": path_or_url, "scope": scope} - - resp = requests.delete(url, params=params, timeout=timeout_s) - body_text = getattr(resp, "text", "") - parsed = None - try: - parsed = resp.json() - except Exception: - parsed = _parse_json_or_none(body_text) if body_text else None - - return { - "http_status": getattr(resp, "status_code", None), - "response_json": parsed if isinstance(parsed, dict) else None, - "response_text": body_text if not isinstance(parsed, dict) else None, - } - - def _build_forward_error( message: str, index_name: str, @@ -384,206 +350,6 @@ def _parse_json_or_none(text: str) -> Optional[Dict[str, Any]]: return None -@dataclass(frozen=True) -class _ForwardContext: - task_id: str - request_id: str - start_time: float - source: str - index_name: str - source_type: str - original_filename: Optional[str] - - -def _init_forward_context( - *, - task_id: str, - request_id: str, - start_time: float, - source: str, - index_name: str, - source_type: str, - original_filename: Optional[str], -) -> _ForwardContext: - return _ForwardContext( - task_id=task_id, - request_id=request_id, - start_time=start_time, - source=source, - index_name=index_name, - source_type=source_type, - original_filename=original_filename, - ) - - -def _is_forward_task_cancelled(ctx: _ForwardContext) -> bool: - try: - redis_service = get_redis_service() - return bool(redis_service.is_task_cancelled(ctx.task_id)) - except Exception as exc: - logger.warning( - f"[{ctx.request_id}] FORWARD TASK: Failed to check cancellation flag for task {ctx.task_id}: " - f"{exc}" - ) - return False - - -def _build_forward_cancelled_result(ctx: _ForwardContext) -> Dict[str, Any]: - return { - 'task_id': ctx.task_id, - 'source': ctx.source, - 'index_name': ctx.index_name, - 'original_filename': ctx.original_filename, - 'chunks_stored': 0, - 'storage_time': 0, - 'es_result': { - "success": False, - "message": "Indexing cancelled because document was deleted.", - "total_indexed": 0, - "total_submitted": 0, - }, - } - - -def _load_forward_chunks( - self: Task, - *, - processed_data: Dict[str, Any], - original_source: str, - original_index_name: str, - filename: Optional[str], -) -> Tuple[Optional[List[Dict[str, Any]]], bool, str, str, Optional[str]]: - chunks = processed_data.get('chunks') - split_async = bool(processed_data.get('split_async')) - - # If chunks are not in payload, try loading from Redis via the redis_key - if (not chunks) and processed_data.get('redis_key'): - redis_key = processed_data.get('redis_key') - if not REDIS_BACKEND_URL: - raise Exception(json.dumps({ - "message": "REDIS_BACKEND_URL not configured to retrieve chunks", - "index_name": original_index_name, - "task_name": "forward", - "source": original_source, - "original_filename": filename - }, ensure_ascii=False)) - try: - import redis - client = redis.Redis.from_url( - REDIS_BACKEND_URL, decode_responses=True) - ready_key = f"{redis_key}:ready" - if split_async: - ready_flag = client.get(ready_key) - if not ready_flag: - retry_num = getattr(self.request, 'retries', 0) - logger.info( - f"[{self.request.id}] FORWARD TASK: Async split not ready for key {redis_key}. Retry {retry_num + 1}/{ASYNC_SPLIT_RETRY_MAX} in {FORWARD_REDIS_RETRY_DELAY_S}s") - raise self.retry( - countdown=FORWARD_REDIS_RETRY_DELAY_S, - max_retries=ASYNC_SPLIT_RETRY_MAX, - exc=Exception(json.dumps({ - "message": "Async split not ready; will retry", - "index_name": original_index_name, - "task_name": "forward", - "source": original_source, - "original_filename": filename - }, ensure_ascii=False)) - ) - cached = client.get(redis_key) - if cached: - try: - logger.debug( - f"[{self.request.id}] FORWARD TASK: Retrieved Redis key '{redis_key}', payload_length={len(cached)}") - chunks = json.loads(cached) - except json.JSONDecodeError as jde: - # Log raw prefix to help diagnose incorrect writes - raw_preview = cached[:120] if isinstance( - cached, str) else str(type(cached)) - logger.error( - f"[{self.request.id}] FORWARD TASK: JSON decode error for key '{redis_key}': {str(jde)}; raw_prefix={raw_preview!r}") - raise - else: - if split_async: - retry_num = getattr(self.request, 'retries', 0) - logger.info( - f"[{self.request.id}] FORWARD TASK: Async split ready but chunks missing for key {redis_key}. Retry {retry_num + 1}/{ASYNC_SPLIT_RETRY_MAX} in {FORWARD_REDIS_RETRY_DELAY_S}s") - raise self.retry( - countdown=FORWARD_REDIS_RETRY_DELAY_S, - max_retries=ASYNC_SPLIT_RETRY_MAX, - exc=Exception(json.dumps({ - "message": "Async split ready but chunks missing; will retry", - "index_name": original_index_name, - "task_name": "forward", - "source": original_source, - "original_filename": filename - }, ensure_ascii=False)) - ) - # No busy-wait: release the worker slot and retry later - retry_num = getattr(self.request, 'retries', 0) - logger.info( - f"[{self.request.id}] FORWARD TASK: Chunks not yet available for key {redis_key}. Retry {retry_num + 1}/{FORWARD_REDIS_RETRY_MAX} in {FORWARD_REDIS_RETRY_DELAY_S}s") - raise self.retry( - countdown=FORWARD_REDIS_RETRY_DELAY_S, - max_retries=FORWARD_REDIS_RETRY_MAX, - exc=Exception(json.dumps({ - "message": "Chunks not ready in Redis; will retry", - "index_name": original_index_name, - "task_name": "forward", - "source": original_source, - "original_filename": filename - }, ensure_ascii=False)) - ) - except Retry: - raise - except Exception as exc: - raise Exception(json.dumps({ - "message": f"Failed to retrieve chunks from Redis: {str(exc)}", - "index_name": original_index_name, - "task_name": "forward", - "source": original_source, - "original_filename": filename - }, ensure_ascii=False)) - - if processed_data.get('source'): - original_source = processed_data.get('source') - if processed_data.get('index_name'): - original_index_name = processed_data.get('index_name') - if processed_data.get('original_filename'): - filename = processed_data.get('original_filename') - - logger.info( - f"[{self.request.id}] FORWARD TASK: Received data for source '{original_source}' with {len(chunks) if chunks else 'None'} chunks") - - if chunks is None: - raise Exception(json.dumps({ - "message": "No chunks received for forwarding", - "index_name": original_index_name, - "task_name": "forward", - "source": original_source, - "original_filename": filename - }, ensure_ascii=False)) - if len(chunks) == 0: - if split_async and processed_data.get('redis_key'): - retry_num = getattr(self.request, 'retries', 0) - logger.info( - f"[{self.request.id}] FORWARD TASK: Empty chunks while waiting for async split. Retry {retry_num + 1}/{ASYNC_SPLIT_RETRY_MAX} in {FORWARD_REDIS_RETRY_DELAY_S}s") - raise self.retry( - countdown=FORWARD_REDIS_RETRY_DELAY_S, - max_retries=ASYNC_SPLIT_RETRY_MAX, - exc=Exception(json.dumps({ - "message": "Chunks not ready in Redis (empty); will retry", - "index_name": original_index_name, - "task_name": "forward", - "source": original_source, - "original_filename": filename - }, ensure_ascii=False)) - ) - logger.warning( - f"[{self.request.id}] FORWARD TASK: Empty chunks list received for source {original_source}") - - return chunks, split_async, original_source, original_index_name, filename - - def _extract_error_code_from_es_response( parsed_body: Optional[Dict[str, Any]], text: str, @@ -638,7 +404,7 @@ async def _post(): try: connector = aiohttp.TCPConnector(verify_ssl=False) timeout = aiohttp.ClientTimeout(total=600) - + request_params: Dict[str, str] = {} if large_mode: @@ -657,8 +423,7 @@ async def _post(): parsed_body = _parse_json_or_none(text) if status >= 400: - error_code = _extract_error_code_from_es_response( - parsed_body, text) + error_code = _extract_error_code_from_es_response(parsed_body, text) if error_code: raise Exception(json.dumps({ "error_code": error_code @@ -743,8 +508,7 @@ def get_actor(self) -> Any: if not self.actors: actor = self._create_and_warm_actor() if actor is None: - raise RuntimeError( - "Global actor pool is empty and actor warm-up failed") + raise RuntimeError("Global actor pool is empty and actor warm-up failed") self.actors.append(actor) idx = self.rr_index % len(self.actors) self.rr_index += 1 @@ -788,12 +552,10 @@ def prewarm_ray_actors(target_size: Optional[int] = None) -> int: """ Ensure a global shared pool of warm Ray actors exists for low-latency task execution. """ - desired = RAY_GLOBAL_ACTOR_POOL_SIZE if target_size is None else max( - 0, int(target_size)) + desired = RAY_GLOBAL_ACTOR_POOL_SIZE if target_size is None else max(0, int(target_size)) manager = _get_or_create_global_pool_manager() current_after = ray.get( - manager.ensure_pool.remote( - desired=desired, max_allowed=_estimate_parallel_parts()) + manager.ensure_pool.remote(desired=desired, max_allowed=_estimate_parallel_parts()) ) logger.info( f"Global Ray actor pool ready: current={current_after}, desired={desired}" @@ -816,7 +578,6 @@ def _get_split_actor() -> Any: """ return get_ray_actor() - class LoggingTask(Task): """Base task class with enhanced logging""" @@ -884,8 +645,7 @@ def process_part( "chunks_count": len(chunks), } except Exception as e: - logger.error( - f"[process_part] Failed to process part for '{filename}': {str(e)}") + logger.error(f"[process_part] Failed to process part for '{filename}': {str(e)}") return { "part_redis_key": part_redis_key, "chunks_count": 0, @@ -1399,8 +1159,7 @@ def process( fetch_start = time.perf_counter() file_stream = get_file_stream(source) if file_stream is None: - raise FileNotFoundError( - f"Unable to fetch file from URL: {source}") + raise FileNotFoundError(f"Unable to fetch file from URL: {source}") file_data = file_stream.read() fetch_elapsed = time.perf_counter() - fetch_start logger.info( @@ -1449,8 +1208,7 @@ def process( if cached: cached_chunks = json.loads(cached) if isinstance(cached_chunks, list): - image_metadata_chunk_count = _count_image_metadata_chunks( - cached_chunks) + image_metadata_chunk_count = _count_image_metadata_chunks(cached_chunks) except Exception as image_count_exc: logger.warning( f"[{self.request.id}] PROCESS TASK: Failed counting image metadata chunks for async split: {image_count_exc}") @@ -1474,17 +1232,17 @@ def process( self.update_state( state=states.SUCCESS, meta={ - 'chunks_count': chunk_count, - 'processing_time': elapsed_time, - 'source': source, - 'index_name': index_name, - 'original_filename': original_filename, - 'task_name': 'process', - 'stage': 'text_extracted', - 'file_size_mb': file_size_mb, - 'processing_speed_mb_s': file_size_mb / elapsed_time if file_size_mb > 0 and elapsed_time > 0 else 0 - } - ) + 'chunks_count': chunk_count, + 'processing_time': elapsed_time, + 'source': source, + 'index_name': index_name, + 'original_filename': original_filename, + 'task_name': 'process', + 'stage': 'text_extracted', + 'file_size_mb': file_size_mb, + 'processing_speed_mb_s': file_size_mb / elapsed_time if file_size_mb > 0 and elapsed_time > 0 else 0 + } + ) logger.info( f"[{self.request.id}] PROCESS TASK: Processing complete, waiting for forward task") @@ -1650,34 +1408,165 @@ def forward( filename = original_filename try: - ctx = _init_forward_context( - task_id=task_id, - request_id=str(self.request.id), - start_time=start_time, - source=source, - index_name=index_name, - source_type=source_type, - original_filename=original_filename, - ) - - # Before doing any heavy work, check whether this task has been explicitly cancelled. - if _is_forward_task_cancelled(ctx): - logger.info( - f"[{self.request.id}] FORWARD TASK: Detected cancellation flag for task {task_id}; " - f"skipping chunk forwarding for source '{source}' in index '{index_name}'." + # Before doing any heavy work, check whether this task has been + # explicitly cancelled (for example, because the user deleted the + # document from the knowledge base configuration page). + try: + redis_service = get_redis_service() + if redis_service.is_task_cancelled(task_id): + logger.info( + f"[{self.request.id}] FORWARD TASK: Detected cancellation flag for task {task_id}; " + f"skipping chunk forwarding for source '{source}' in index '{index_name}'." + ) + # Treat this as a graceful early exit. We still return a + # structured payload so callers can consider the task done. + return { + 'task_id': task_id, + 'source': source, + 'index_name': index_name, + 'original_filename': original_filename, + 'chunks_stored': 0, + 'storage_time': 0, + 'es_result': { + "success": False, + "message": "Indexing cancelled because document was deleted.", + "total_indexed": 0, + "total_submitted": 0, + }, + } + except Exception as cancel_check_exc: + logger.warning( + f"[{self.request.id}] FORWARD TASK: Failed to check cancellation flag for task {task_id}: " + f"{cancel_check_exc}" ) - return _build_forward_cancelled_result(ctx) - - chunks, split_async, original_source, original_index_name, filename = _load_forward_chunks( - self, - processed_data=processed_data, - original_source=original_source, - original_index_name=original_index_name, - filename=filename, - ) + + chunks = processed_data.get('chunks') + split_async = bool(processed_data.get('split_async')) + # If chunks are not in payload, try loading from Redis via the redis_key + if (not chunks) and processed_data.get('redis_key'): + redis_key = processed_data.get('redis_key') + if not REDIS_BACKEND_URL: + raise Exception(json.dumps({ + "message": "REDIS_BACKEND_URL not configured to retrieve chunks", + "index_name": original_index_name, + "task_name": "forward", + "source": original_source, + "original_filename": filename + }, ensure_ascii=False)) + try: + import redis + client = redis.Redis.from_url( + REDIS_BACKEND_URL, decode_responses=True) + ready_key = f"{redis_key}:ready" + if split_async: + ready_flag = client.get(ready_key) + if not ready_flag: + retry_num = getattr(self.request, 'retries', 0) + logger.info( + f"[{self.request.id}] FORWARD TASK: Async split not ready for key {redis_key}. Retry {retry_num + 1}/{ASYNC_SPLIT_RETRY_MAX} in {FORWARD_REDIS_RETRY_DELAY_S}s") + raise self.retry( + countdown=FORWARD_REDIS_RETRY_DELAY_S, + max_retries=ASYNC_SPLIT_RETRY_MAX, + exc=Exception(json.dumps({ + "message": "Async split not ready; will retry", + "index_name": original_index_name, + "task_name": "forward", + "source": original_source, + "original_filename": filename + }, ensure_ascii=False)) + ) + cached = client.get(redis_key) + if cached: + try: + logger.debug( + f"[{self.request.id}] FORWARD TASK: Retrieved Redis key '{redis_key}', payload_length={len(cached)}") + chunks = json.loads(cached) + except json.JSONDecodeError as jde: + # Log raw prefix to help diagnose incorrect writes + raw_preview = cached[:120] if isinstance( + cached, str) else str(type(cached)) + logger.error( + f"[{self.request.id}] FORWARD TASK: JSON decode error for key '{redis_key}': {str(jde)}; raw_prefix={raw_preview!r}") + raise + else: + if split_async: + retry_num = getattr(self.request, 'retries', 0) + logger.info( + f"[{self.request.id}] FORWARD TASK: Async split ready but chunks missing for key {redis_key}. Retry {retry_num + 1}/{ASYNC_SPLIT_RETRY_MAX} in {FORWARD_REDIS_RETRY_DELAY_S}s") + raise self.retry( + countdown=FORWARD_REDIS_RETRY_DELAY_S, + max_retries=ASYNC_SPLIT_RETRY_MAX, + exc=Exception(json.dumps({ + "message": "Async split ready but chunks missing; will retry", + "index_name": original_index_name, + "task_name": "forward", + "source": original_source, + "original_filename": filename + }, ensure_ascii=False)) + ) + # No busy-wait: release the worker slot and retry later + retry_num = getattr(self.request, 'retries', 0) + logger.info( + f"[{self.request.id}] FORWARD TASK: Chunks not yet available for key {redis_key}. Retry {retry_num + 1}/{FORWARD_REDIS_RETRY_MAX} in {FORWARD_REDIS_RETRY_DELAY_S}s") + raise self.retry( + countdown=FORWARD_REDIS_RETRY_DELAY_S, + max_retries=FORWARD_REDIS_RETRY_MAX, + exc=Exception(json.dumps({ + "message": "Chunks not ready in Redis; will retry", + "index_name": original_index_name, + "task_name": "forward", + "source": original_source, + "original_filename": filename + }, ensure_ascii=False)) + ) + except Retry: + raise + except Exception as exc: + raise Exception(json.dumps({ + "message": f"Failed to retrieve chunks from Redis: {str(exc)}", + "index_name": original_index_name, + "task_name": "forward", + "source": original_source, + "original_filename": filename + }, ensure_ascii=False)) + if processed_data.get('source'): + original_source = processed_data.get('source') + if processed_data.get('index_name'): + original_index_name = processed_data.get('index_name') + if processed_data.get('original_filename'): + filename = processed_data.get('original_filename') + logger.info( + f"[{self.request.id}] FORWARD TASK: Received data for source '{original_source}' with {len(chunks) if chunks else 'None'} chunks") # Calculate total chunks for progress tracking total_chunks = len(chunks) if chunks else 0 + + if chunks is None: + raise Exception(json.dumps({ + "message": "No chunks received for forwarding", + "index_name": original_index_name, + "task_name": "forward", + "source": original_source, + "original_filename": original_filename + }, ensure_ascii=False)) + if len(chunks) == 0: + if split_async and processed_data.get('redis_key'): + retry_num = getattr(self.request, 'retries', 0) + logger.info( + f"[{self.request.id}] FORWARD TASK: Empty chunks while waiting for async split. Retry {retry_num + 1}/{ASYNC_SPLIT_RETRY_MAX} in {FORWARD_REDIS_RETRY_DELAY_S}s") + raise self.retry( + countdown=FORWARD_REDIS_RETRY_DELAY_S, + max_retries=ASYNC_SPLIT_RETRY_MAX, + exc=Exception(json.dumps({ + "message": "Chunks not ready in Redis (empty); will retry", + "index_name": original_index_name, + "task_name": "forward", + "source": original_source, + "original_filename": filename + }, ensure_ascii=False)) + ) + logger.warning( + f"[{self.request.id}] FORWARD TASK: Empty chunks list received for source {original_source}") formatted_chunks = [] # Compute once per file to avoid repeated IO/MinIO calls inside loop file_size = get_file_size(source_type, original_source) if isinstance( @@ -1868,7 +1757,6 @@ def forward( logger.info( f"[{self.request.id}] FORWARD TASK: Successfully stored {len(chunks)} chunks to index {original_index_name} in {end_time - start_time:.2f}s") - return { 'task_id': task_id, 'source': original_source, @@ -1951,106 +1839,9 @@ def forward( raise -@app.task( - bind=True, - base=LoggingTask, - name="data_process.tasks.cleanup_source", - queue="forward_q", -) -def cleanup_source(self, forward_result: Dict[str, Any]) -> Dict[str, Any]: - """ - Conditionally delete the MinIO source file after successful indexing. - - If the knowledge base is configured with preserve_source_file=false, call: - DELETE /indices/{index_name}/documents?path_or_url=...&scope=source_only - """ - index_name = (forward_result or {}).get("index_name") - source = (forward_result or {}).get("source") - - cleanup_info: Dict[str, Any] = { - "attempted": False, - "skipped_reason": None, - "success": None, - "http_status": None, - "response": None, - "error": None, - } - - if not index_name or not source: - cleanup_info["skipped_reason"] = "missing_index_name_or_source" - forward_result = dict(forward_result or {}) - forward_result["source_cleanup"] = cleanup_info - return forward_result - - try: - record = get_knowledge_record({"index_name": index_name}) or {} - preserve_source_file = record.get("preserve_source_file", True) - except Exception as exc: - logger.warning( - "[%s] CLEANUP TASK: Failed to load knowledge config for index '%s': %s", - getattr(self.request, "id", "unknown"), - index_name, - exc, - ) - cleanup_info["skipped_reason"] = "knowledge_record_lookup_failed" - forward_result = dict(forward_result or {}) - forward_result["source_cleanup"] = cleanup_info - return forward_result - - if preserve_source_file: - cleanup_info["skipped_reason"] = "preserve_source_file_true" - forward_result = dict(forward_result or {}) - forward_result["source_cleanup"] = cleanup_info - return forward_result - - cleanup_info["attempted"] = True - try: - resp = _delete_source_file_via_http_sync( - base_url=ELASTICSEARCH_SERVICE, - index_name=index_name, - path_or_url=source, - scope="source_only", - ) - cleanup_info["http_status"] = resp.get("http_status") - cleanup_info["response"] = ( - resp.get("response_json") - if resp.get("response_json") is not None - else resp.get("response_text") - ) - - ok = False - if isinstance(resp.get("response_json"), dict): - ok = bool(resp["response_json"].get("status") == "success") - elif resp.get("http_status") and 200 <= int(resp["http_status"]) < 300: - ok = True - - cleanup_info["success"] = ok - if not ok: - logger.warning( - "[%s] CLEANUP TASK: Source-only delete did not succeed. index='%s' source='%s' http_status=%s", - getattr(self.request, "id", "unknown"), - index_name, - source, - cleanup_info["http_status"], - ) - except Exception as exc: - cleanup_info["success"] = False - cleanup_info["error"] = str(exc) - logger.warning( - "[%s] CLEANUP TASK: Source-only delete failed. index='%s' source='%s' error=%s", - getattr(self.request, "id", "unknown"), - index_name, - source, - exc, - ) - - forward_result = dict(forward_result or {}) - forward_result["source_cleanup"] = cleanup_info - return forward_result - - -def submit_process_forward_chain( - *, +@app.task(bind=True, base=LoggingTask, name='data_process.tasks.process_and_forward') +def process_and_forward( + self, source: str, source_type: str, chunking_strategy: str, @@ -2058,14 +1849,30 @@ def submit_process_forward_chain( original_filename: Optional[str] = None, authorization: Optional[str] = None, embedding_model_id: Optional[int] = None, - tenant_id: Optional[str] = None, + tenant_id: Optional[str] = None ) -> str: """ - Build and enqueue a Celery chain: process -> forward. + Combined task that chains processing and forwarding + + This task delegates to a chain of process -> forward + + Args: + source: Source file path, URL, or text content + source_type: source of the file("local", "minio") + chunking_strategy: Strategy for chunking the document + index_name: Name of the index to store documents + original_filename: The original name of the file + authorization: Authorization header for API calls + embedding_model_id: Embedding model ID for chunk size configuration + tenant_id: Tenant ID for retrieving model configuration Returns: - Celery chain task ID, or empty string if enqueue failed. + Task ID of the chain """ + logger.info( + f"Starting processing chain for {source}, original_filename={original_filename}, strategy={chunking_strategy}, index={index_name}, model_id={embedding_model_id}") + + # Create a task chain task_chain = chain( process.s( source=source, @@ -2082,64 +1889,18 @@ def submit_process_forward_chain( source_type=source_type, original_filename=original_filename, authorization=authorization - ).set(queue='forward_q'), - cleanup_source.s().set(queue='forward_q'), + ).set(queue='forward_q') ) + # Execute the chain result = task_chain.apply_async() if result is None or not hasattr(result, 'id') or result.id is None: logger.error( "Celery chain apply_async() did not return a valid result or result.id") return "" - return result.id - + logger.info(f"Created task chain ID: {result.id}") -@app.task(bind=True, base=LoggingTask, name='data_process.tasks.process_and_forward') -def process_and_forward( - self, - source: str, - source_type: str, - chunking_strategy: str, - index_name: Optional[str] = None, - original_filename: Optional[str] = None, - authorization: Optional[str] = None, - embedding_model_id: Optional[int] = None, - tenant_id: Optional[str] = None -) -> str: - """ - Combined task that chains processing and forwarding - - This task delegates to a chain of process -> forward - - Args: - source: Source file path, URL, or text content - source_type: source of the file("local", "minio") - chunking_strategy: Strategy for chunking the document - index_name: Name of the index to store documents - original_filename: The original name of the file - authorization: Authorization header for API calls - embedding_model_id: Embedding model ID for chunk size configuration - tenant_id: Tenant ID for retrieving model configuration - - Returns: - Task ID of the chain - """ - logger.info( - f"Starting processing chain for {source}, original_filename={original_filename}, strategy={chunking_strategy}, index={index_name}, model_id={embedding_model_id}") - - chain_id = submit_process_forward_chain( - source=source, - source_type=source_type, - chunking_strategy=chunking_strategy, - index_name=index_name, - original_filename=original_filename, - authorization=authorization, - embedding_model_id=embedding_model_id, - tenant_id=tenant_id, - ) - if chain_id: - logger.info(f"Created task chain ID: {chain_id}") - return chain_id + return result.id @app.task(bind=True, base=LoggingTask, name='data_process.tasks.process_sync') diff --git a/backend/database/agent_db.py b/backend/database/agent_db.py index 533659b0f..82696ffab 100644 --- a/backend/database/agent_db.py +++ b/backend/database/agent_db.py @@ -1,10 +1,9 @@ import logging -from typing import List, Optional +from typing import List from sqlalchemy import or_, update from database.client import get_db_session, as_dict, filter_property from database.db_models import AgentInfo, ToolInstance, AgentRelation -from database.agent_version_db import query_current_version_no from consts.const import ASSET_OWNER_TENANT_ID from utils.str_utils import convert_list_to_string @@ -103,40 +102,6 @@ def query_sub_agents_id_list(main_agent_id: int, tenant_id: str, version_no: int return [relation.selected_agent_id for relation in relations] -def query_sub_agent_relations(main_agent_id: int, tenant_id: str, version_no: int = 0) -> List[dict]: - """ - Query sub-agent relations by main agent id, including pinned version info. - Default version_no=0 queries the draft version. - - Args: - main_agent_id: Parent agent ID - tenant_id: Tenant ID - version_no: Version number to filter. Default 0 = draft/editing state - """ - with get_db_session() as session: - query = session.query(AgentRelation).filter( - AgentRelation.parent_agent_id == main_agent_id, - AgentRelation.tenant_id == tenant_id, - AgentRelation.version_no == version_no, - AgentRelation.delete_flag != 'Y') - relations = query.all() - return [as_dict(relation) for relation in relations] - - -def resolve_sub_agent_version_no( - selected_agent_id: int, - selected_agent_version_no: Optional[int], - tenant_id: str, -) -> int: - """ - Resolve the effective version number for a sub-agent relation. - Uses pinned version when set; otherwise falls back to child's current published version. - """ - if selected_agent_version_no is not None: - return selected_agent_version_no - return query_current_version_no(agent_id=selected_agent_id, tenant_id=tenant_id) or 0 - - def clear_agent_new_mark(agent_id: int, tenant_id: str, user_id: str, version_no: int = 0): """ Clear the NEW mark for an agent. @@ -198,7 +163,6 @@ def create_agent(agent_info, tenant_id: str, user_id: str): """ info_with_metadata = dict(agent_info) info_with_metadata.setdefault("max_steps", 15) - info_with_metadata.setdefault("verification_config", None) info_with_metadata.update({ "tenant_id": tenant_id, "version_no": 0, # Default to draft version @@ -237,9 +201,6 @@ def create_agent(agent_info, tenant_id: str, user_id: str): "group_ids": new_agent.group_ids, "is_new": new_agent.is_new, "enable_context_manager": new_agent.enable_context_manager, - "verification_config": new_agent.verification_config, - "greeting_message": new_agent.greeting_message, - "example_questions": new_agent.example_questions, "current_version_no": new_agent.current_version_no, "version_no": new_agent.version_no, "created_by": new_agent.created_by, diff --git a/backend/database/agent_repository_db.py b/backend/database/agent_repository_db.py deleted file mode 100644 index a6bb4f48b..000000000 --- a/backend/database/agent_repository_db.py +++ /dev/null @@ -1,358 +0,0 @@ -import logging -import math -from typing import Any, Dict, List, Optional - -from sqlalchemy import func, or_, update - -from database.client import as_dict, filter_property, get_db_session -from database.db_models import AgentRepository - -logger = logging.getLogger("agent_repository_db") - -# Listing status: NOT_SHARED (未共享), PENDING_REVIEW (待审核), -# REJECTED (审核驳回), SHARED (已共享) -STATUS_NOT_SHARED = "NOT_SHARED" -STATUS_PENDING_REVIEW = "PENDING_REVIEW" -STATUS_REJECTED = "REJECTED" -STATUS_SHARED = "SHARED" - -VALID_REPOSITORY_STATUSES = frozenset({ - STATUS_NOT_SHARED, - STATUS_PENDING_REVIEW, - STATUS_REJECTED, - STATUS_SHARED, -}) - -_UPSERT_IMMUTABLE_FIELDS = frozenset({ - "agent_id", - "agent_repository_id", - "publisher_tenant_id", -}) - -_UPSERT_SNAPSHOT_FIELDS = frozenset({ - "source_version_no", - "name", - "display_name", - "description", - "author", - "category_id", - "tags", - "tool_count", - "version_label", - "agent_info_json", -}) - - -def insert_agent_repository_record( - repository_data: Dict[str, Any], - publisher_tenant_id: str, - publisher_user_id: str, -) -> int: - """Insert a new agent repository listing record.""" - with get_db_session() as session: - payload = { - **repository_data, - "publisher_tenant_id": publisher_tenant_id, - "publisher_user_id": publisher_user_id, - "created_by": publisher_user_id, - "updated_by": publisher_user_id, - "delete_flag": "N", - } - if payload.get("status") is None: - payload["status"] = STATUS_NOT_SHARED - - new_record = AgentRepository( - **filter_property(payload, AgentRepository) - ) - session.add(new_record) - session.flush() - return int(new_record.agent_repository_id) - - -def get_agent_repository_by_id(repository_id: int) -> Optional[dict]: - """Fetch a repository listing by primary key.""" - with get_db_session() as session: - record = session.query(AgentRepository).filter( - AgentRepository.agent_repository_id == repository_id, - AgentRepository.delete_flag != "Y", - ).first() - return as_dict(record) if record else None - - -def get_agent_repository_by_id_and_publisher( - repository_id: int, - publisher_tenant_id: str, -) -> Optional[dict]: - """Fetch a repository listing scoped to the publisher tenant.""" - with get_db_session() as session: - record = session.query(AgentRepository).filter( - AgentRepository.agent_repository_id == repository_id, - AgentRepository.publisher_tenant_id == publisher_tenant_id, - AgentRepository.delete_flag != "Y", - ).first() - return as_dict(record) if record else None - - -def get_agent_repository_by_agent_id(agent_id: int) -> Optional[dict]: - """Fetch an active repository listing by root agent_id.""" - with get_db_session() as session: - record = session.query(AgentRepository).filter( - AgentRepository.agent_id == agent_id, - AgentRepository.delete_flag != "Y", - ).first() - return as_dict(record) if record else None - - -def upsert_agent_repository_record( - repository_data: Dict[str, Any], - publisher_tenant_id: str, - publisher_user_id: str, -) -> tuple[int, bool]: - """Insert or update a repository listing keyed by agent_id. - - When no record exists, inserts a new listing. When a record exists: - - Same source_version_no: updates status (and updated_by) only. - - Different source_version_no: updates all snapshot fields, preserving - agent_id, agent_repository_id, and publisher_tenant_id. - - Returns: - Tuple of (agent_repository_id, is_updated). is_updated is False on insert. - """ - agent_id = repository_data.get("agent_id") - if agent_id is None: - raise ValueError("agent_id is required for repository upsert") - - existing = get_agent_repository_by_agent_id(int(agent_id)) - if not existing: - repository_id = insert_agent_repository_record( - repository_data=repository_data, - publisher_tenant_id=publisher_tenant_id, - publisher_user_id=publisher_user_id, - ) - return repository_id, False - - existing_version = existing.get("source_version_no") - incoming_version = repository_data.get("source_version_no") - repository_id = int(existing["agent_repository_id"]) - - if existing_version == incoming_version: - update_fields: Dict[str, Any] = { - "status": repository_data.get("status", STATUS_NOT_SHARED), - "updated_by": publisher_user_id, - } - else: - update_fields = { - key: repository_data[key] - for key in _UPSERT_SNAPSHOT_FIELDS - if key in repository_data - } - update_fields["publisher_user_id"] = publisher_user_id - update_fields["updated_by"] = publisher_user_id - update_fields["status"] = repository_data.get("status", STATUS_NOT_SHARED) - - with get_db_session() as session: - session.execute( - update(AgentRepository) - .where( - AgentRepository.agent_repository_id == repository_id, - AgentRepository.publisher_tenant_id == publisher_tenant_id, - AgentRepository.delete_flag != "Y", - ) - .values(**update_fields) - ) - return repository_id, True - - -def list_agent_repository_summaries( - *, - status: Optional[str] = None, -) -> List[dict]: - """List all active repository summaries without heavy JSON blobs.""" - with get_db_session() as session: - query = session.query( - AgentRepository.agent_repository_id, - AgentRepository.author, - AgentRepository.name, - AgentRepository.display_name, - AgentRepository.description, - AgentRepository.status, - ).filter( - AgentRepository.delete_flag != "Y", - ) - if status: - query = query.filter(AgentRepository.status == status) - rows = query.order_by(AgentRepository.agent_repository_id.desc()).all() - return [ - { - "agent_repository_id": row.agent_repository_id, - "author": row.author, - "name": row.name, - "display_name": row.display_name, - "description": row.description, - "status": row.status, - } - for row in rows - ] - - -def query_agent_repository_list( - *, - page: int = 1, - page_size: int = 20, - search: Optional[str] = None, - tag: Optional[str] = None, - category_id: Optional[int] = None, - status: Optional[str] = STATUS_SHARED, - publisher_tenant_id: Optional[str] = None, -) -> Dict[str, Any]: - """Query repository listings with offset pagination.""" - page = max(page, 1) - page_size = max(min(page_size, 100), 1) - offset = (page - 1) * page_size - - with get_db_session() as session: - query = session.query(AgentRepository).filter( - AgentRepository.delete_flag != "Y", - ) - - if status: - query = query.filter(AgentRepository.status == status) - if publisher_tenant_id: - query = query.filter( - AgentRepository.publisher_tenant_id == publisher_tenant_id - ) - if category_id is not None: - query = query.filter(AgentRepository.category_id == category_id) - if tag: - query = query.filter(AgentRepository.tags.any(tag)) - if search: - keyword = f"%{search}%" - query = query.filter( - or_( - AgentRepository.name.ilike(keyword), - AgentRepository.display_name.ilike(keyword), - AgentRepository.description.ilike(keyword), - AgentRepository.author.ilike(keyword), - func.array_to_string(AgentRepository.tags, ",").ilike(keyword), - ) - ) - - total = query.count() - rows = ( - query.order_by(AgentRepository.agent_repository_id.desc()) - .offset(offset) - .limit(page_size) - .all() - ) - - total_pages = math.ceil(total / page_size) if total else 0 - return { - "items": [as_dict(row) for row in rows], - "pagination": { - "page": page, - "page_size": page_size, - "total": total, - "total_pages": total_pages, - }, - } - - -def update_agent_repository_by_id( - *, - repository_id: int, - publisher_tenant_id: str, - user_id: str, - updates: Dict[str, Any], -) -> int: - """Update a repository listing owned by the publisher tenant. Returns affected row count.""" - allowed_fields = { - "display_name", - "description", - "author", - "category_id", - "tags", - "tool_count", - "version_label", - "source_version_no", - "agent_info_json", - "status", - } - update_fields = { - key: value - for key, value in updates.items() - if key in allowed_fields - } - if not update_fields: - return 0 - - update_fields["updated_by"] = user_id - - with get_db_session() as session: - result = session.execute( - update(AgentRepository) - .where( - AgentRepository.agent_repository_id == repository_id, - AgentRepository.publisher_tenant_id == publisher_tenant_id, - AgentRepository.delete_flag != "Y", - ) - .values(**update_fields) - ) - return int(result.rowcount or 0) - - -def update_agent_repository_status_by_id( - *, - repository_id: int, - status: str, - user_id: str, -) -> int: - """Update repository listing status by primary key. Returns affected row count.""" - with get_db_session() as session: - result = session.execute( - update(AgentRepository) - .where( - AgentRepository.agent_repository_id == repository_id, - AgentRepository.delete_flag != "Y", - ) - .values(status=status, updated_by=user_id) - ) - return int(result.rowcount or 0) - - -def soft_delete_agent_repository_by_id( - *, - repository_id: int, - publisher_tenant_id: str, - user_id: str, -) -> int: - """Soft-delete a repository listing owned by the publisher tenant.""" - with get_db_session() as session: - result = session.execute( - update(AgentRepository) - .where( - AgentRepository.agent_repository_id == repository_id, - AgentRepository.publisher_tenant_id == publisher_tenant_id, - AgentRepository.delete_flag != "Y", - ) - .values(delete_flag="Y", updated_by=user_id) - ) - return int(result.rowcount or 0) - - -def list_agent_repository_by_publisher( - publisher_tenant_id: str, - *, - publisher_user_id: Optional[str] = None, -) -> List[dict]: - """List all repository listings published by a tenant.""" - with get_db_session() as session: - query = session.query(AgentRepository).filter( - AgentRepository.publisher_tenant_id == publisher_tenant_id, - AgentRepository.delete_flag != "Y", - ) - if publisher_user_id: - query = query.filter( - AgentRepository.publisher_user_id == publisher_user_id - ) - rows = query.order_by(AgentRepository.agent_repository_id.desc()).all() - return [as_dict(row) for row in rows] diff --git a/backend/database/cas_session_db.py b/backend/database/cas_session_db.py deleted file mode 100644 index 57d1aa8ea..000000000 --- a/backend/database/cas_session_db.py +++ /dev/null @@ -1,134 +0,0 @@ -""" -Database operations for CAS-backed web sessions. -""" - -from datetime import datetime -from typing import Any, Dict, Optional - -from database.client import as_dict, get_db_session -from database.db_models import UserCasSession - -CAS_SESSION_ACTIVE = "active" -CAS_SESSION_REVOKED = "revoked" - - -def create_cas_session( - *, - session_id: str, - user_id: str, - cas_user_id: str, - expires_at: datetime, - cas_session_index: Optional[str] = None, -) -> Dict[str, Any]: - with get_db_session() as session: - record = UserCasSession( - session_id=session_id, - user_id=user_id, - cas_user_id=cas_user_id, - cas_session_index=cas_session_index, - status=CAS_SESSION_ACTIVE, - expires_at=expires_at, - created_by=user_id, - updated_by=user_id, - ) - session.add(record) - session.flush() - return as_dict(record) - - -def get_cas_session_by_session_id(session_id: str) -> Optional[Dict[str, Any]]: - if not session_id: - return None - with get_db_session() as session: - result = ( - session.query(UserCasSession) - .filter( - UserCasSession.session_id == session_id, - UserCasSession.delete_flag == "N", - ) - .first() - ) - return as_dict(result) if result else None - - -def is_cas_session_active(session_id: str) -> bool: - if not session_id: - return False - with get_db_session() as session: - result = ( - session.query(UserCasSession) - .filter( - UserCasSession.session_id == session_id, - UserCasSession.status == CAS_SESSION_ACTIVE, - UserCasSession.expires_at > datetime.now(), - UserCasSession.delete_flag == "N", - ) - .first() - ) - return result is not None - - -def revoke_cas_session_by_session_id(session_id: str, actor: str = "cas") -> int: - if not session_id: - return 0 - with get_db_session() as session: - result = ( - session.query(UserCasSession) - .filter( - UserCasSession.session_id == session_id, - UserCasSession.status == CAS_SESSION_ACTIVE, - UserCasSession.delete_flag == "N", - ) - .update( - { - "status": CAS_SESSION_REVOKED, - "revoked_at": datetime.now(), - "updated_by": actor, - } - ) - ) - return result - - -def revoke_cas_sessions_by_user_id(cas_user_id: str, actor: str = "cas") -> int: - if not cas_user_id: - return 0 - with get_db_session() as session: - result = ( - session.query(UserCasSession) - .filter( - UserCasSession.cas_user_id == cas_user_id, - UserCasSession.status == CAS_SESSION_ACTIVE, - UserCasSession.delete_flag == "N", - ) - .update( - { - "status": CAS_SESSION_REVOKED, - "revoked_at": datetime.now(), - "updated_by": actor, - } - ) - ) - return result - - -def revoke_cas_session_by_index(cas_session_index: str, actor: str = "cas") -> int: - if not cas_session_index: - return 0 - with get_db_session() as session: - result = ( - session.query(UserCasSession) - .filter( - UserCasSession.cas_session_index == cas_session_index, - UserCasSession.status == CAS_SESSION_ACTIVE, - UserCasSession.delete_flag == "N", - ) - .update( - { - "status": CAS_SESSION_REVOKED, - "revoked_at": datetime.now(), - "updated_by": actor, - } - ) - ) - return result diff --git a/backend/database/conversation_db.py b/backend/database/conversation_db.py index 2d06bb9be..18c0ee9fc 100644 --- a/backend/database/conversation_db.py +++ b/backend/database/conversation_db.py @@ -1016,71 +1016,3 @@ def get_message_id_by_index(conversation_id: int, message_index: int) -> Optiona result = session.execute(stmt).scalar() return result - - -def get_latest_assistant_message_id(conversation_id: int, user_id: Optional[str] = None) -> Optional[int]: - """ - Get the most recent assistant message ID for a conversation. - - Args: - conversation_id: Conversation ID (integer) - user_id: Optional user ID for ownership check - - Returns: - Optional[int]: The latest assistant message ID, or None if not found - """ - with get_db_session() as session: - conversation_id = int(conversation_id) - - stmt = select(ConversationMessage.message_id).where( - ConversationMessage.conversation_id == conversation_id, - ConversationMessage.delete_flag == 'N', - ConversationMessage.message_role == 'assistant' - ).order_by(desc(ConversationMessage.message_index)).limit(1) - - if user_id: - stmt = stmt.join( - ConversationRecord, - ConversationMessage.conversation_id == ConversationRecord.conversation_id - ).where(ConversationRecord.created_by == user_id) - - result = session.execute(stmt).scalar() - return result - - -def update_message_minio_files(message_id: int, skill_file_uploads: List[Dict[str, Any]]) -> bool: - """ - Merge skill file uploads into an existing message's minio_files field. - - Args: - message_id: Message ID to update - skill_file_uploads: List of skill file upload metadata dicts to append - - Returns: - bool: True if the message was updated, False if the message was not found - """ - with get_db_session() as session: - message_id = int(message_id) - - stmt = select(ConversationMessage).where( - ConversationMessage.message_id == message_id, - ConversationMessage.delete_flag == 'N' - ) - record = session.scalars(stmt).first() - if not record: - return False - - existing = record.minio_files - if existing: - try: - if isinstance(existing, str): - existing = json.loads(existing) - except (json.JSONDecodeError, TypeError): - existing = [] - else: - existing = [] - - existing.extend(skill_file_uploads) - record.minio_files = json.dumps(existing, ensure_ascii=False) - - return True diff --git a/backend/database/db_models.py b/backend/database/db_models.py index 5450b5f74..b779266c9 100644 --- a/backend/database/db_models.py +++ b/backend/database/db_models.py @@ -15,8 +15,6 @@ _TENANT_ID_DOC = "Tenant ID for multi-tenancy isolation" # Base class for tables without audit fields - - class SimpleTableBase(DeclarativeBase): pass @@ -299,16 +297,13 @@ class AgentInfo(TableBase): agent_id = Column(Integer, Sequence( "ag_tenant_agent_t_agent_id_seq", schema=SCHEMA), nullable=False, primary_key=True, autoincrement=True, doc="ID") - version_no = Column(Integer, default=0, nullable=False, primary_key=True, - doc="Version number. 0 = draft/editing state, >=1 = published snapshot") + version_no = Column(Integer, default=0, nullable=False, primary_key=True, doc="Version number. 0 = draft/editing state, >=1 = published snapshot") name = Column(String(100), doc="Agent name") display_name = Column(String(100), doc="Agent display name") description = Column(Text, doc="Description") author = Column(String(100), doc="Agent author") - model_name = Column( - String(100), doc="[DEPRECATED] Name of the model used, use model_id instead") - model_id = Column( - Integer, doc="Model ID, foreign key reference to model_record_t.model_id") + model_name = Column(String(100), doc="[DEPRECATED] Name of the model used, use model_id instead") + model_id = Column(Integer, doc="Model ID, foreign key reference to model_record_t.model_id") max_steps = Column(Integer, doc="Maximum number of steps") duty_prompt = Column(Text, doc="Duty prompt content") constraint_prompt = Column(Text, doc="Constraint prompt content") @@ -320,22 +315,15 @@ class AgentInfo(TableBase): Boolean, doc="Whether to provide the running summary to the manager agent") business_description = Column( Text, doc="Manually entered by the user to describe the entire business process") - business_logic_model_name = Column( - String(100), doc="Model name used for business logic prompt generation") - business_logic_model_id = Column( - Integer, doc="Model ID used for business logic prompt generation, foreign key reference to model_record_t.model_id") - prompt_template_id = Column( - Integer, doc="Prompt template ID used for business logic prompt generation") - prompt_template_name = Column(String( - 100), doc="Prompt template name used for business logic prompt generation") + business_logic_model_name = Column(String(100), doc="Model name used for business logic prompt generation") + business_logic_model_id = Column(Integer, doc="Model ID used for business logic prompt generation, foreign key reference to model_record_t.model_id") + prompt_template_id = Column(Integer, doc="Prompt template ID used for business logic prompt generation") + prompt_template_name = Column(String(100), doc="Prompt template name used for business logic prompt generation") group_ids = Column(String, doc="Agent group IDs list") is_new = Column(Boolean, default=False, doc="Whether this agent is marked as new for the user") current_version_no = Column(Integer, nullable=True, doc="Current published version number. NULL means no version published yet") ingroup_permission = Column(String(30), doc="In-group permission: EDIT, READ_ONLY, PRIVATE") enable_context_manager = Column(Boolean, default=False, doc="Whether to enable context management (compression) for this agent") - verification_config = Column(JSONB, doc="Layered ReAct self-verification configuration") - greeting_message = Column(Text, doc="Agent greeting message displayed on chat initial screen") - example_questions = Column(JSONB, doc="List of example questions for starting a conversation with this agent") class PromptTemplate(TableBase): @@ -364,15 +352,12 @@ class PromptTemplate(TableBase): template_id = Column(Integer, Sequence( "ag_prompt_template_t_template_id_seq", schema=SCHEMA), primary_key=True, nullable=False, autoincrement=True, doc="Prompt template ID") - template_name = Column(String(100), nullable=False, - doc="Prompt template name") + template_name = Column(String(100), nullable=False, doc="Prompt template name") description = Column(String(500), doc="Prompt template description") - template_type = Column(String(50), nullable=False, - default="agent_generate", doc="Prompt template type") + template_type = Column(String(50), nullable=False, default="agent_generate", doc="Prompt template type") tenant_id = Column(String(100), nullable=False, doc="Tenant ID") user_id = Column(String(100), nullable=False, doc="User ID") - template_content_zh = Column( - JSONB, nullable=False, doc="Chinese prompt template content") + template_content_zh = Column(JSONB, nullable=False, doc="Chinese prompt template content") template_content_en = Column(JSONB, doc="English prompt template content") @@ -396,8 +381,7 @@ class ToolInstance(TableBase): user_id = Column(String(100), doc="User ID") tenant_id = Column(String(100), doc="Tenant ID") enabled = Column(Boolean, doc="Enabled") - version_no = Column(Integer, default=0, primary_key=True, nullable=False, - doc="Version number. 0 = draft/editing state, >=1 = published snapshot") + version_no = Column(Integer, default=0, primary_key=True, nullable=False, doc="Version number. 0 = draft/editing state, >=1 = published snapshot") class KnowledgeRecord(TableBase): @@ -413,25 +397,18 @@ class KnowledgeRecord(TableBase): knowledge_name = Column(String(100), doc="User-facing knowledge base name") knowledge_describe = Column(String(3000), doc="Knowledge base description") knowledge_sources = Column(String(300), doc="Knowledge base sources") - embedding_model_name = Column(String( - 200), doc="Embedding model name, used to record the embedding model used by the knowledge base") - embedding_model_id = Column( - Integer, doc="Embedding model ID, foreign key reference to model_record_t.model_id") + embedding_model_name = Column(String(200), doc="Embedding model name, used to record the embedding model used by the knowledge base") + embedding_model_id = Column(Integer, doc="Embedding model ID, foreign key reference to model_record_t.model_id") tenant_id = Column(String(100), doc="Tenant ID") group_ids = Column(String, doc="Knowledge base group IDs list") ingroup_permission = Column( String(30), doc="In-group permission: EDIT, READ_ONLY, PRIVATE") summary_frequency = Column(String(10), nullable=True, - doc="Auto-summary frequency: '3h', '5h', '1d', '1w', or NULL (disabled)") + doc="Auto-summary frequency: '3h', '5h', '1d', '1w', or NULL (disabled)") last_summary_time = Column(TIMESTAMP(timezone=False), nullable=True, - doc="Timestamp of last summary generation") + doc="Timestamp of last summary generation") last_doc_update_time = Column(TIMESTAMP(timezone=False), nullable=True, - doc="Timestamp of last document add/delete operation") - preserve_source_file = Column( - Boolean, - default=True, - doc="Whether to preserve uploaded source documents after vectorization", - ) + doc="Timestamp of last document add/delete operation") class TenantConfig(TableBase): @@ -504,8 +481,7 @@ class McpRecord(TableBase): doc="Custom HTTP headers as JSON object for MCP server requests", default=None, ) - source = Column( - String(30), doc="Source type: local/mcp_registry/community") + source = Column(String(30), doc="Source type: local/mcp_registry/community") registry_json = Column(JSONB, doc="Full MCP registry server.json snapshot") config_json = Column(JSON, doc="MCP config data") enabled = Column(Boolean, default=True, doc="Enabled") @@ -533,13 +509,11 @@ class McpCommunityRecord(TableBase): source = Column(String(30), doc="Source type, fixed to community") version = Column(String(50), doc="MCP version") registry_json = Column(JSONB, doc="Full MCP metadata JSON") - transport_type = Column( - String(30), doc="Transport type: http/sse/container") + transport_type = Column(String(30), doc="Transport type: http/sse/container") config_json = Column(JSON, doc="Public-shareable MCP configuration JSON") tags = Column(ARRAY(Text), doc="Tags") description = Column(Text, doc="Description") - class UserTenant(TableBase): """ User and tenant relationship table @@ -551,8 +525,7 @@ class UserTenant(TableBase): primary_key=True, nullable=False, doc="User tenant relationship ID, unique primary key") user_id = Column(String(100), nullable=False, doc="User ID") tenant_id = Column(String(100), nullable=False, doc="Tenant ID") - user_role = Column( - String(30), doc="User role: SUPER_ADMIN, ADMIN, DEV, USER") + user_role = Column(String(30), doc="User role: SUPER_ADMIN, ADMIN, DEV, USER") user_email = Column(String(255), doc="User email address") @@ -563,18 +536,11 @@ class AgentRelation(TableBase): __tablename__ = "ag_agent_relation_t" __table_args__ = {"schema": SCHEMA} - relation_id = Column(Integer, Sequence("ag_agent_relation_t_relation_id_seq", schema=SCHEMA), - primary_key=True, nullable=False, doc="Relationship ID, primary key") - selected_agent_id = Column( - Integer, primary_key=True, doc="Selected agent ID") + relation_id = Column(Integer, Sequence("ag_agent_relation_t_relation_id_seq", schema=SCHEMA), primary_key=True, nullable=False, doc="Relationship ID, primary key") + selected_agent_id = Column(Integer, primary_key=True, doc="Selected agent ID") parent_agent_id = Column(Integer, doc="Parent agent ID") tenant_id = Column(String(100), doc="Tenant ID") - version_no = Column(Integer, default=0, nullable=False, - doc="Version number. 0 = draft/editing state, >=1 = published snapshot") - selected_agent_version_no = Column( - Integer, nullable=True, - doc="Pinned version of selected_agent_id. NULL = runtime fallback to child current_version_no", - ) + version_no = Column(Integer, default=0, nullable=False, doc="Version number. 0 = draft/editing state, >=1 = published snapshot") class PartnerMappingId(TableBase): @@ -690,51 +656,13 @@ class AgentVersion(TableBase): primary_key=True, nullable=False, doc=_PRIMARY_KEY_DOC) tenant_id = Column(String(100), nullable=False, doc="Tenant ID") agent_id = Column(Integer, nullable=False, doc="Agent ID") - version_no = Column(Integer, nullable=False, - doc="Version number, starts from 1. Does not include 0 (draft)") - version_name = Column( - String(100), doc="User-defined version name for display") + version_no = Column(Integer, nullable=False, doc="Version number, starts from 1. Does not include 0 (draft)") + version_name = Column(String(100), doc="User-defined version name for display") release_note = Column(Text, doc="Release notes / publish remarks") - source_version_no = Column( - Integer, doc="Source version number. If this version is a rollback, record the source version") - source_type = Column(String( - 30), doc="Source type: NORMAL (normal publish) / ROLLBACK (rollback and republish)") - status = Column(String(30), default="RELEASED", - doc="Version status: RELEASED / DISABLED / ARCHIVED") - is_a2a = Column(Boolean, default=False, - doc="Whether this version is published as an A2A Server agent") - - -class AgentRepository(TableBase): - """ - Agent repository (marketplace) table. Frozen snapshot of a published agent tree for sharing. - """ - __tablename__ = "ag_agent_repository_t" - __table_args__ = {"schema": SCHEMA} - - agent_repository_id = Column(BigInteger, Sequence("ag_agent_repository_t_agent_repository_id_seq", schema=SCHEMA), - primary_key=True, nullable=False, doc="Agent repository listing ID, unique primary key") - publisher_tenant_id = Column(String(100), nullable=False, doc="Publisher tenant ID") - publisher_user_id = Column(String(100), nullable=False, doc="Publisher user ID") - agent_id = Column(Integer, nullable=False, - doc="Root agent ID from ag_tenant_agent_t; upsert key") - source_version_no = Column(Integer, nullable=False, - doc="Published version number frozen at share time") - name = Column(String(100), nullable=False, - doc="Root agent programmatic name for display and search") - display_name = Column(String(100), doc="Root agent display name") - description = Column(Text, doc="Root agent description") - author = Column(String(100), doc="Agent author") - category_id = Column(Integer, doc="Optional marketplace category ID") - tags = Column(ARRAY(Text), doc="Marketplace tags") - tool_count = Column(Integer, - doc="Total tool count across all agents in the bundle (display only)") - version_label = Column(String(100), - doc="Repository entry version label for display (e.g. v1.0)") - agent_info_json = Column(JSONB, nullable=False, - doc="Frozen ExportAndImportDataFormat snapshot with optional skills") - status = Column(String(30), default="NOT_SHARED", - doc="Listing status: NOT_SHARED (未共享) / PENDING_REVIEW (待审核) / REJECTED (审核驳回) / SHARED (已共享)") + source_version_no = Column(Integer, doc="Source version number. If this version is a rollback, record the source version") + source_type = Column(String(30), doc="Source type: NORMAL (normal publish) / ROLLBACK (rollback and republish)") + status = Column(String(30), default="RELEASED", doc="Version status: RELEASED / DISABLED / ARCHIVED") + is_a2a = Column(Boolean, default=False, doc="Whether this version is published as an A2A Server agent") class UserTokenInfo(TableBase): @@ -747,8 +675,7 @@ class UserTokenInfo(TableBase): token_id = Column(Integer, Sequence("user_token_info_t_token_id_seq", schema=SCHEMA), primary_key=True, nullable=False, doc="Token ID, unique primary key") access_key = Column(String(100), nullable=False, doc="Access Key (AK)") - user_id = Column(String(100), nullable=False, - doc="User ID who owns this token") + user_id = Column(String(100), nullable=False, doc="User ID who owns this token") class UserTokenUsageLog(TableBase): @@ -760,21 +687,16 @@ class UserTokenUsageLog(TableBase): token_usage_id = Column(Integer, Sequence("user_token_usage_log_t_token_usage_id_seq", schema=SCHEMA), primary_key=True, nullable=False, doc="Token usage log ID, unique primary key") - token_id = Column(Integer, nullable=False, - doc="Foreign key to user_token_info_t.token_id") - call_function_name = Column( - String(100), doc="API function name being called") - related_id = Column( - Integer, doc="Related resource ID (e.g., conversation_id)") - meta_data = Column( - JSONB, doc="Additional metadata for this usage log entry, stored as JSON") + token_id = Column(Integer, nullable=False, doc="Foreign key to user_token_info_t.token_id") + call_function_name = Column(String(100), doc="API function name being called") + related_id = Column(Integer, doc="Related resource ID (e.g., conversation_id)") + meta_data = Column(JSONB, doc="Additional metadata for this usage log entry, stored as JSON") class UserOAuthAccount(TableBase): __tablename__ = "user_oauth_account_t" __table_args__ = ( - UniqueConstraint("provider", "provider_user_id", - name="uq_oauth_provider_user"), + UniqueConstraint("provider", "provider_user_id", name="uq_oauth_provider_user"), {"schema": SCHEMA}, ) @@ -792,38 +714,11 @@ class UserOAuthAccount(TableBase): provider_user_id = Column( String(200), nullable=False, doc="User ID from the OAuth provider" ) - provider_email = Column( - String(255), doc="Email address from the OAuth provider") - provider_username = Column( - String(200), doc="Display name from the OAuth provider") + provider_email = Column(String(255), doc="Email address from the OAuth provider") + provider_username = Column(String(200), doc="Display name from the OAuth provider") tenant_id = Column(String(100), doc="Tenant ID at time of linking") -class UserCasSession(TableBase): - __tablename__ = "user_cas_session_t" - __table_args__ = ( - Index("ix_user_cas_session_session_id", "session_id"), - Index("ix_user_cas_session_user_id", "user_id"), - Index("ix_user_cas_session_cas_user_id", "cas_user_id"), - {"schema": SCHEMA}, - ) - - cas_session_id = Column( - Integer, - Sequence("user_cas_session_t_cas_session_id_seq", schema=SCHEMA), - primary_key=True, - nullable=False, - doc="CAS session record ID", - ) - session_id = Column(String(100), nullable=False, unique=True, doc="JWT session ID") - user_id = Column(String(100), nullable=False, doc="Supabase user UUID") - cas_user_id = Column(String(200), nullable=False, doc="User ID from CAS") - cas_session_index = Column(String(500), doc="CAS SessionIndex or service ticket") - status = Column(String(30), nullable=False, default="active", doc="active/revoked") - expires_at = Column(TIMESTAMP(timezone=False), nullable=False, doc="Session expiration time") - revoked_at = Column(TIMESTAMP(timezone=False), doc="Revocation time") - - class SkillInfo(TableBase): """ Skill information table - stores skill metadata and content. @@ -833,17 +728,13 @@ class SkillInfo(TableBase): skill_id = Column(Integer, Sequence("ag_skill_info_t_skill_id_seq", schema=SCHEMA), primary_key=True, nullable=False, autoincrement=True, doc="Skill ID") - skill_name = Column(String(100), nullable=False, - unique=True, doc="Unique skill name") - tenant_id = Column(String(100), nullable=True, - doc="Tenant ID for multi-tenancy. NULL for pre-existing skills.") + skill_name = Column(String(100), nullable=False, unique=True, doc="Unique skill name") + tenant_id = Column(String(100), nullable=True, doc="Tenant ID for multi-tenancy. NULL for pre-existing skills.") skill_description = Column(String(1000), doc="Skill description") skill_tags = Column(JSON, doc="Skill tags as JSON array") skill_content = Column(Text, doc="Skill content in markdown format") - config_schemas = Column( - JSON, doc="Parameter metadata from config/schema.yaml") - config_values = Column( - JSON, doc="Runtime parameter values from config/config.yaml") + config_schemas = Column(JSON, doc="Parameter metadata from config/schema.yaml") + config_values = Column(JSON, doc="Runtime parameter values from config/config.yaml") source = Column(String(30), nullable=False, default="official", doc="Skill source: official, custom, etc.") @@ -857,10 +748,8 @@ class SkillToolRelation(TableBase): rel_id = Column(Integer, Sequence("ag_skill_tools_rel_t_rel_id_seq", schema=SCHEMA), primary_key=True, nullable=False, autoincrement=True, doc="Relation ID") - skill_id = Column(Integer, nullable=False, - doc="Foreign key to ag_skill_info_t.skill_id") - tool_id = Column(Integer, nullable=False, - doc="Foreign key to ag_tool_info_t.tool_id") + skill_id = Column(Integer, nullable=False, doc="Foreign key to ag_skill_info_t.skill_id") + tool_id = Column(Integer, nullable=False, doc="Foreign key to ag_tool_info_t.tool_id") class SkillInstance(TableBase): @@ -879,19 +768,14 @@ class SkillInstance(TableBase): nullable=False, doc="Skill instance ID" ) - skill_id = Column(Integer, nullable=False, - doc="Foreign key to ag_skill_info_t.skill_id") + skill_id = Column(Integer, nullable=False, doc="Foreign key to ag_skill_info_t.skill_id") agent_id = Column(Integer, nullable=False, doc="Agent ID") user_id = Column(String(100), doc="User ID") tenant_id = Column(String(100), doc="Tenant ID") - enabled = Column(Boolean, default=True, - doc="Whether this skill is enabled for the agent") - version_no = Column(Integer, default=0, primary_key=True, nullable=False, - doc="Version number. 0 = draft/editing state, >=1 = published snapshot") - config_values = Column( - JSON, doc="Per-agent runtime parameter values (mirrors ag_tool_instance_t.params)") - config_schemas = Column( - JSON, doc="Per-agent parameter schema overrides from config/schema.yaml") + enabled = Column(Boolean, default=True, doc="Whether this skill is enabled for the agent") + version_no = Column(Integer, default=0, primary_key=True, nullable=False, doc="Version number. 0 = draft/editing state, >=1 = published snapshot") + config_values = Column(JSON, doc="Per-agent runtime parameter values (mirrors ag_tool_instance_t.params)") + config_schemas = Column(JSON, doc="Per-agent parameter schema overrides from config/schema.yaml") class OuterApiService(TableBase): @@ -904,16 +788,13 @@ class OuterApiService(TableBase): id = Column(BigInteger, Sequence("ag_outer_api_services_id_seq", schema=SCHEMA), primary_key=True, nullable=False, doc="Service ID, unique primary key") - mcp_service_name = Column(String(100), nullable=False, - doc="MCP service name (unique identifier per tenant)") + mcp_service_name = Column(String(100), nullable=False, doc="MCP service name (unique identifier per tenant)") description = Column(Text, doc="Service description from OpenAPI info") openapi_json = Column(JSONB, doc="Complete OpenAPI JSON specification") server_url = Column(String(500), doc="Base URL of the REST API server") headers_template = Column(JSONB, doc="Default headers template as JSON") - tenant_id = Column(String(100), nullable=False, - doc="Tenant ID for multi-tenancy") - is_available = Column(Boolean, default=True, - doc="Whether the service is available") + tenant_id = Column(String(100), nullable=False, doc="Tenant ID for multi-tenancy") + is_available = Column(Boolean, default=True, doc="Whether the service is available") # Alias for backward compatibility @@ -928,37 +809,27 @@ class A2ANacosConfig(TableBase): __tablename__ = "ag_a2a_nacos_config_t" __table_args__ = {"schema": SCHEMA} - id = Column(BigInteger, primary_key=True, - autoincrement=True, doc=_PRIMARY_KEY_DOC) - config_id = Column(String(64), unique=True, nullable=False, - doc="Unique config identifier for API reference") + id = Column(BigInteger, primary_key=True, autoincrement=True, doc=_PRIMARY_KEY_DOC) + config_id = Column(String(64), unique=True, nullable=False, doc="Unique config identifier for API reference") # Nacos connection - nacos_addr = Column(String(512), nullable=False, - doc="Nacos server address, e.g., http://nacos-server:8848") - nacos_username = Column( - String(100), doc="Nacos username for authentication") - nacos_password = Column( - String(256), doc="Nacos password, encrypted at rest") + nacos_addr = Column(String(512), nullable=False, doc="Nacos server address, e.g., http://nacos-server:8848") + nacos_username = Column(String(100), doc="Nacos username for authentication") + nacos_password = Column(String(256), doc="Nacos password, encrypted at rest") # Discovery scope - namespace_id = Column(String(100), default="public", - doc="Nacos namespace for service discovery") + namespace_id = Column(String(100), default="public", doc="Nacos namespace for service discovery") # Metadata - name = Column(String(100), nullable=False, - doc="Display name for this Nacos config") + name = Column(String(100), nullable=False, doc="Display name for this Nacos config") description = Column(Text, doc="Description of this Nacos configuration") # Tenant isolation - tenant_id = Column(String(100), nullable=False, - doc="Tenant ID for multi-tenancy") + tenant_id = Column(String(100), nullable=False, doc="Tenant ID for multi-tenancy") # Status - is_active = Column(Boolean, default=True, - doc="Whether this Nacos config is active") - last_scan_at = Column(TIMESTAMP(timezone=False), - doc="Last time a scan was performed using this config") + is_active = Column(Boolean, default=True, doc="Whether this Nacos config is active") + last_scan_at = Column(TIMESTAMP(timezone=False), doc="Last time a scan was performed using this config") class A2AExternalAgent(TableBase): @@ -969,49 +840,39 @@ class A2AExternalAgent(TableBase): __tablename__ = "ag_a2a_external_agent_t" __table_args__ = {"schema": SCHEMA} - id = Column(BigInteger, primary_key=True, - autoincrement=True, doc=_PRIMARY_KEY_DOC) + id = Column(BigInteger, primary_key=True, autoincrement=True, doc=_PRIMARY_KEY_DOC) # Agent metadata (cached from Agent Card) - name = Column(String(255), nullable=False, - doc="Agent name from Agent Card") + name = Column(String(255), nullable=False, doc="Agent name from Agent Card") description = Column(Text, doc="Agent description from Agent Card") - version = Column( - String(50), doc="Agent version from Agent Card, e.g., 1.2.0") + version = Column(String(50), doc="Agent version from Agent Card, e.g., 1.2.0") # Primary interface (extracted from supportedInterfaces for quick access) # In A2A 1.0, this should store the http-json-rpc URL - agent_url = Column(String(512), nullable=False, - doc="Primary A2A endpoint URL (http-json-rpc by default)") + agent_url = Column(String(512), nullable=False, doc="Primary A2A endpoint URL (http-json-rpc by default)") # Protocol type for calling this agent: JSONRPC, HTTP+JSON, GRPC - protocol_type = Column(String(20), default=PROTOCOL_JSONRPC, - doc="Protocol type for calling this agent") + protocol_type = Column(String(20), default=PROTOCOL_JSONRPC, doc="Protocol type for calling this agent") # Capabilities - streaming = Column(Boolean, default=False, - doc="Whether this agent supports SSE streaming") + streaming = Column(Boolean, default=False, doc="Whether this agent supports SSE streaming") # All supported interfaces (full JSON array from Agent Card) # Format: [{protocolBinding, url, protocolVersion}, ...] supported_interfaces = Column(JSON, doc="All supported interfaces array") # Source information - source_type = Column(String(20), nullable=False, - doc="Discovery source: url or nacos") + source_type = Column(String(20), nullable=False, doc="Discovery source: url or nacos") # For URL mode source_url = Column(String(512), doc="Direct URL to agent card") # For Nacos mode - nacos_config_id = Column( - String(64), doc="Reference to Nacos config used for discovery") - nacos_agent_name = Column( - String(255), doc="Original name used for Nacos query") + nacos_config_id = Column(String(64), doc="Reference to Nacos config used for discovery") + nacos_agent_name = Column(String(255), doc="Original name used for Nacos query") # Base URL for infrastructure health checks - base_url = Column(String( - 512), doc="Base URL for health checks (service root address), e.g., http://agent:8080") + base_url = Column(String(512), doc="Base URL for health checks (service root address), e.g., http://agent:8080") # Tenant isolation tenant_id = Column(String(100), nullable=False, doc=_TENANT_ID_DOC) @@ -1020,18 +881,13 @@ class A2AExternalAgent(TableBase): raw_card = Column(JSON, doc="Full original Agent Card JSON from discovery") # Cache management - cached_at = Column(TIMESTAMP(timezone=False), - doc="Timestamp when Agent Card was cached") - cache_expires_at = Column( - TIMESTAMP(timezone=False), doc="Timestamp when cache expires") + cached_at = Column(TIMESTAMP(timezone=False), doc="Timestamp when Agent Card was cached") + cache_expires_at = Column(TIMESTAMP(timezone=False), doc="Timestamp when cache expires") # Health check status - is_available = Column(Boolean, default=True, - doc="Whether this agent is currently reachable") - last_check_at = Column(TIMESTAMP(timezone=False), - doc="Last health check timestamp") - last_check_result = Column( - String(50), doc="Last health check result: OK, ERROR, TIMEOUT") + is_available = Column(Boolean, default=True, doc="Whether this agent is currently reachable") + last_check_at = Column(TIMESTAMP(timezone=False), doc="Last health check timestamp") + last_check_result = Column(String(50), doc="Last health check result: OK, ERROR, TIMEOUT") class A2AExternalAgentRelation(TableBase): @@ -1049,23 +905,19 @@ class A2AExternalAgentRelation(TableBase): {"schema": SCHEMA}, ) - id = Column(BigInteger, primary_key=True, - autoincrement=True, doc=_PRIMARY_KEY_DOC) + id = Column(BigInteger, primary_key=True, autoincrement=True, doc=_PRIMARY_KEY_DOC) # Local agent (parent) - local_agent_id = Column(Integer, nullable=False, - doc="Local parent agent ID") + local_agent_id = Column(Integer, nullable=False, doc="Local parent agent ID") # External A2A agent (sub-agent) - FK to ag_a2a_external_agent_t.id - external_agent_id = Column( - BigInteger, nullable=False, doc="External A2A agent ID (FK to ag_a2a_external_agent_t.id)") + external_agent_id = Column(BigInteger, nullable=False, doc="External A2A agent ID (FK to ag_a2a_external_agent_t.id)") # Tenant isolation tenant_id = Column(String(100), nullable=False, doc=_TENANT_ID_DOC) # Status - is_enabled = Column(Boolean, default=True, - doc="Whether this relation is active") + is_enabled = Column(Boolean, default=True, doc="Whether this relation is active") class A2AServerAgent(TableBase): @@ -1076,8 +928,7 @@ class A2AServerAgent(TableBase): __tablename__ = "ag_a2a_server_agent_t" __table_args__ = {"schema": SCHEMA} - id = Column(BigInteger, primary_key=True, - autoincrement=True, doc=_PRIMARY_KEY_DOC) + id = Column(BigInteger, primary_key=True, autoincrement=True, doc=_PRIMARY_KEY_DOC) # Link to local agent agent_id = Column(Integer, nullable=False, doc="Local agent ID") @@ -1087,44 +938,35 @@ class A2AServerAgent(TableBase): tenant_id = Column(String(100), nullable=False, doc=_TENANT_ID_DOC) # Generated endpoint ID - endpoint_id = Column(String(64), unique=True, - nullable=False, doc="Generated endpoint ID") + endpoint_id = Column(String(64), unique=True, nullable=False, doc="Generated endpoint ID") # Basic info (extracted from local agent, can be overridden) - name = Column(String(255), nullable=False, - doc="Agent name exposed in Agent Card") + name = Column(String(255), nullable=False, doc="Agent name exposed in Agent Card") description = Column(Text, doc="Agent description exposed in Agent Card") version = Column(String(50), doc="Agent version exposed in Agent Card") # Primary endpoint URL (http-json-rpc by default) - agent_url = Column( - String(512), doc="Primary A2A endpoint URL (http-json-rpc by default)") + agent_url = Column(String(512), doc="Primary A2A endpoint URL (http-json-rpc by default)") # Capabilities - streaming = Column(Boolean, default=False, - doc="Whether this agent supports SSE streaming") + streaming = Column(Boolean, default=False, doc="Whether this agent supports SSE streaming") # All supported interfaces (A2A 1.0 compliant) # Format: [{protocolBinding, url, protocolVersion}, ...] - supported_interfaces = Column( - JSON, doc="All supported interfaces: [{protocolBinding, url, protocolVersion}, ...]") + supported_interfaces = Column(JSON, doc="All supported interfaces: [{protocolBinding, url, protocolVersion}, ...]") # Agent Card customization (partial overrides only) - card_overrides = Column( - JSON, doc="User customizations for Agent Card (partial override)") + card_overrides = Column(JSON, doc="User customizations for Agent Card (partial override)") # A2A Server status - is_enabled = Column(Boolean, default=False, - doc="Whether A2A Server is enabled for this agent") + is_enabled = Column(Boolean, default=False, doc="Whether A2A Server is enabled for this agent") # Raw Agent Card (generated from settings, for debugging) raw_card = Column(JSON, doc="Generated Agent Card JSON (for debugging)") # Publishing timestamps - published_at = Column(TIMESTAMP(timezone=False), - doc="Timestamp when A2A Server was last enabled") - unpublished_at = Column(TIMESTAMP(timezone=False), - doc="Timestamp when A2A Server was disabled") + published_at = Column(TIMESTAMP(timezone=False), doc="Timestamp when A2A Server was last enabled") + unpublished_at = Column(TIMESTAMP(timezone=False), doc="Timestamp when A2A Server was disabled") class A2ATask(SimpleTableBase): @@ -1137,8 +979,7 @@ class A2ATask(SimpleTableBase): # Core identifiers (following A2A spec) id = Column(String(64), primary_key=True, doc="Task ID (A2A spec: taskId)") - context_id = Column( - String(64), doc="Context ID for grouping related tasks") + context_id = Column(String(64), doc="Context ID for grouping related tasks") # Endpoint and caller info endpoint_id = Column(String(64), nullable=False, doc="Endpoint ID") @@ -1149,21 +990,16 @@ class A2ATask(SimpleTableBase): raw_request = Column(JSON, doc="Original A2A request payload") # Task state (following A2A TaskState enum) - task_state = Column(String(50), nullable=False, server_default="TASK_STATE_SUBMITTED", - doc="Task state: TASK_STATE_SUBMITTED, TASK_STATE_WORKING, TASK_STATE_COMPLETED, TASK_STATE_FAILED, TASK_STATE_CANCELED, TASK_STATE_INPUT_REQUIRED, TASK_STATE_REJECTED, TASK_STATE_AUTH_REQUIRED") - state_timestamp = Column(TIMESTAMP(timezone=False), - doc="Task state last update timestamp") + task_state = Column(String(50), nullable=False, server_default="TASK_STATE_SUBMITTED", doc="Task state: TASK_STATE_SUBMITTED, TASK_STATE_WORKING, TASK_STATE_COMPLETED, TASK_STATE_FAILED, TASK_STATE_CANCELED, TASK_STATE_INPUT_REQUIRED, TASK_STATE_REJECTED, TASK_STATE_AUTH_REQUIRED") + state_timestamp = Column(TIMESTAMP(timezone=False), doc="Task state last update timestamp") # Task result result_data = Column(JSON, doc="Task final result data") # Timestamps - create_time = Column(TIMESTAMP(timezone=False), - server_default=func.now(), doc="Task creation timestamp") - update_time = Column(TIMESTAMP(timezone=False), server_default=func.now( - ), onupdate=func.now(), doc="Task last update timestamp") - completed_at = Column(TIMESTAMP(timezone=False), - doc="Task completion timestamp") + create_time = Column(TIMESTAMP(timezone=False), server_default=func.now(), doc="Task creation timestamp") + update_time = Column(TIMESTAMP(timezone=False), server_default=func.now(), onupdate=func.now(), doc="Task last update timestamp") + completed_at = Column(TIMESTAMP(timezone=False), doc="Task completion timestamp") class A2AMessage(SimpleTableBase): @@ -1175,30 +1011,23 @@ class A2AMessage(SimpleTableBase): __table_args__ = {"schema": SCHEMA} # Core identifiers (following A2A spec) - message_id = Column(String(64), primary_key=True, - doc="Message ID (A2A spec: messageId)") - task_id = Column(String(64), nullable=True, - doc="Task ID this message belongs to (nullable for standalone/simple requests)") + message_id = Column(String(64), primary_key=True, doc="Message ID (A2A spec: messageId)") + task_id = Column(String(64), nullable=True, doc="Task ID this message belongs to (nullable for standalone/simple requests)") # Message attributes - message_index = Column(Integer, nullable=False, - doc="Order of message in the conversation") - role = Column(String(20), nullable=False, - doc="Message sender role: user or agent") + message_index = Column(Integer, nullable=False, doc="Order of message in the conversation") + role = Column(String(20), nullable=False, doc="Message sender role: user or agent") # Message content (following A2A Part structure) - parts = Column(JSON, nullable=False, - doc="Message parts following A2A Part structure") + parts = Column(JSON, nullable=False, doc="Message parts following A2A Part structure") meta_data = Column(JSON, doc="Optional metadata") extensions = Column(JSON, doc="Extension URI list") # References to other tasks (optional) - reference_task_ids = Column( - JSON, doc="Referenced task IDs array for multi-turn scenarios") + reference_task_ids = Column(JSON, doc="Referenced task IDs array for multi-turn scenarios") # Timestamp - create_time = Column(TIMESTAMP( - timezone=False), server_default=func.now(), doc="Message creation timestamp") + create_time = Column(TIMESTAMP(timezone=False), server_default=func.now(), doc="Message creation timestamp") class A2AArtifact(SimpleTableBase): @@ -1210,19 +1039,15 @@ class A2AArtifact(SimpleTableBase): # Core identifiers (following A2A spec) id = Column(String(64), primary_key=True, doc="Internal primary key") - artifact_id = Column(String(64), nullable=False, - doc="Artifact ID (A2A spec: artifactId)") - task_id = Column(String(64), nullable=False, - doc="Task ID this artifact belongs to") + artifact_id = Column(String(64), nullable=False, doc="Artifact ID (A2A spec: artifactId)") + task_id = Column(String(64), nullable=False, doc="Task ID this artifact belongs to") # Artifact attributes name = Column(String(255), doc="Human-readable artifact name") description = Column(Text, doc="Artifact description") - parts = Column(JSON, nullable=False, - doc="Artifact parts following A2A Part structure") + parts = Column(JSON, nullable=False, doc="Artifact parts following A2A Part structure") meta_data = Column(JSON, doc="Artifact metadata") extensions = Column(JSON, doc="Extension URI list") # Timestamp - create_time = Column(TIMESTAMP( - timezone=False), server_default=func.now(), doc="Artifact creation timestamp") + create_time = Column(TIMESTAMP(timezone=False), server_default=func.now(), doc="Artifact creation timestamp") diff --git a/backend/database/knowledge_db.py b/backend/database/knowledge_db.py index 8fc60d6bd..9a8b1c8c1 100644 --- a/backend/database/knowledge_db.py +++ b/backend/database/knowledge_db.py @@ -34,7 +34,6 @@ def create_knowledge_record(query: Dict[str, Any]) -> Dict[str, Any]: - user_id: Optional user ID for created_by and updated_by fields - tenant_id: Optional tenant ID for created_by and updated_by fields - embedding_model_name: embedding model name for the knowledge base - - preserve_source_file: whether to preserve uploaded source documents (optional) Returns: Dict[str, Any]: Dictionary with at least 'knowledge_id' and 'index_name' @@ -58,7 +57,6 @@ def create_knowledge_record(query: Dict[str, Any]) -> Dict[str, Any]: "knowledge_name": knowledge_name, "group_ids": convert_list_to_string(group_ids) if isinstance(group_ids, list) else group_ids, "ingroup_permission": query.get("ingroup_permission"), - "preserve_source_file": query.get("preserve_source_file", True), } # For backward compatibility: if caller explicitly provides index_name, @@ -119,16 +117,11 @@ def upsert_knowledge_record(query: Dict[str, Any]) -> Dict[str, Any]: if existing_record: # Update existing record - existing_record.knowledge_name = query.get( - 'knowledge_name') or query.get('index_name') - existing_record.knowledge_describe = query.get( - 'knowledge_describe', '') - existing_record.knowledge_sources = query.get( - 'knowledge_sources', 'elasticsearch') - existing_record.embedding_model_name = query.get( - 'embedding_model_name') - existing_record.embedding_model_id = query.get( - 'embedding_model_id') + existing_record.knowledge_name = query.get('knowledge_name') or query.get('index_name') + existing_record.knowledge_describe = query.get('knowledge_describe', '') + existing_record.knowledge_sources = query.get('knowledge_sources', 'elasticsearch') + existing_record.embedding_model_name = query.get('embedding_model_name') + existing_record.embedding_model_id = query.get('embedding_model_id') existing_record.updated_by = query.get('user_id') existing_record.update_time = func.current_timestamp() @@ -190,7 +183,7 @@ def update_knowledge_record(query: Dict[str, Any]) -> bool: # Update group IDs if query.get("group_ids") is not None: record.group_ids = query["group_ids"] - + # Update timestamp and user if query.get("user_id"): record.updated_by = query["user_id"] @@ -258,17 +251,15 @@ def get_knowledge_record(query: Optional[Dict[str, Any]] = None) -> Dict[str, An # Support both index_name and knowledge_name queries if 'index_name' in query: - db_query = db_query.filter( - KnowledgeRecord.index_name == query['index_name']) + db_query = db_query.filter(KnowledgeRecord.index_name == query['index_name']) elif 'knowledge_name' in query: - db_query = db_query.filter( - KnowledgeRecord.knowledge_name == query['knowledge_name']) + db_query = db_query.filter(KnowledgeRecord.knowledge_name == query['knowledge_name']) # Add tenant_id filter only if it is provided in the query if 'tenant_id' in query and query['tenant_id'] is not None: db_query = db_query.filter( KnowledgeRecord.tenant_id == query['tenant_id']) - + result = db_query.first() if result: diff --git a/backend/database/user_tenant_db.py b/backend/database/user_tenant_db.py index b147eac49..f1294f8a7 100644 --- a/backend/database/user_tenant_db.py +++ b/backend/database/user_tenant_db.py @@ -75,37 +75,6 @@ def insert_user_tenant(user_id: str, tenant_id: str, user_role: str = "USER", us session.add(user_tenant) -def upsert_user_tenant(user_id: str, tenant_id: str, user_role: str = "USER", user_email: str = None) -> Dict[str, Any]: - """ - Create or update the active user-tenant relationship for an external identity login. - """ - with get_db_session() as session: - result = session.query(UserTenant).filter( - UserTenant.user_id == user_id, - UserTenant.delete_flag == "N" - ).first() - - if result: - result.tenant_id = tenant_id - result.user_role = user_role - if user_email is not None: - result.user_email = user_email - result.updated_by = user_id - else: - result = UserTenant( - user_id=user_id, - tenant_id=tenant_id, - user_role=user_role, - user_email=user_email, - created_by=user_id, - updated_by=user_id - ) - session.add(result) - - session.flush() - return as_dict(result) - - def get_users_by_tenant_id(tenant_id: str, page: Optional[int] = 1, page_size: Optional[int] = 20, sort_by: str = "created_at", sort_order: str = "desc") -> Dict[str, Any]: """ diff --git a/backend/mcp_service.py b/backend/mcp_service.py index 4629d42ad..0d8ab4c1b 100644 --- a/backend/mcp_service.py +++ b/backend/mcp_service.py @@ -70,7 +70,7 @@ async def run(self, arguments: Dict[str, Any]) -> Any: nexent_mcp = FastMCP(name="nexent_mcp") -nexent_mcp.mount(local_mcp_service, local_mcp_service.name) +nexent_mcp.mount(local_mcp_service.name, local_mcp_service) _openapi_mcp_services: Dict[str, FastMCP] = {} @@ -188,8 +188,7 @@ def _sanitize_function_name(name: str) -> str: def register_openapi_service( service_name: str, openapi_json: Dict[str, Any], - server_url: str, - headers_template: Dict[str, str], + server_url: str ) -> bool: """ Register an OpenAPI service using FastMCP.from_openapi(). @@ -223,7 +222,7 @@ def register_openapi_service( openapi_spec["servers"] = [{"url": server_url}] # Create HTTP client for the underlying REST API - client = httpx.AsyncClient(base_url=server_url, timeout=120.0, headers=headers_template) + client = httpx.AsyncClient(base_url=server_url, timeout=30.0) # Create FastMCP instance from OpenAPI spec mcp_server = FastMCP.from_openapi( @@ -240,7 +239,7 @@ def register_openapi_service( _openapi_mcp_services[service_name] = mcp_server # Mount to the main MCP server - nexent_mcp.mount(mcp_server, service_name) + nexent_mcp.mount(service_name, mcp_server) logger.info(f"Registered OpenAPI service: {service_name}") return True @@ -321,14 +320,13 @@ def refresh_openapi_services_by_tenant(tenant_id: str) -> Dict[str, Any]: service_name = service.get("mcp_service_name") openapi_json = service.get("openapi_json") server_url = service.get("server_url") - headers_template = service.get("headers_template") if not openapi_json: logger.warning(f"Service '{service_name}' has no OpenAPI JSON, skipping") skipped_count += 1 continue - if register_openapi_service(service_name, openapi_json, server_url, headers_template): + if register_openapi_service(service_name, openapi_json, server_url): registered_count += 1 else: skipped_count += 1 @@ -396,7 +394,6 @@ def refresh_single_openapi_service(service_name: str, tenant_id: str) -> Dict[st # Re-register with fresh data openapi_json = service_data.get("openapi_json") server_url = service_data.get("server_url") - headers_template = service_data.get("headers_template") if not openapi_json: logger.warning(f"Service '{service_name}' has no OpenAPI JSON") @@ -406,7 +403,7 @@ def refresh_single_openapi_service(service_name: str, tenant_id: str) -> Dict[st "error": "No OpenAPI JSON found" } - success = register_openapi_service(service_name, openapi_json, server_url, headers_template) + success = register_openapi_service(service_name, openapi_json, server_url) return { "status": "refreshed" if success else "error", "service_name": service_name, diff --git a/backend/prompts/managed_system_prompt_template_en.yaml b/backend/prompts/managed_system_prompt_template_en.yaml index 62e16e946..5c2893c39 100644 --- a/backend/prompts/managed_system_prompt_template_en.yaml +++ b/backend/prompts/managed_system_prompt_template_en.yaml @@ -1,6 +1,6 @@ system_prompt: |- ### Basic Information - You are {{APP_NAME}}, {{APP_DESCRIPTION}} + You are {{APP_NAME}}, {{APP_DESCRIPTION}}, it is {{time|default('current time')}} now {%- if memory_list and memory_list|length > 0 %} ### Contextual Memory @@ -66,11 +66,6 @@ system_prompt: |- - Note that executed code is not visible to users. If users need to see the code, use 'code' for displaying code. - **IMPORTANT**: After code execution, the system will return content with "Observation:" marker (this is the real execution result). Please continue your next thinking based on these real results. **Do NOT fabricate observation results before code execution.** - 3. Self-verification: - - After critical events (tool calls, retrieval results, code execution, and final-answer preparation), the system may run explicit verification. - - If verification reports errors, insufficient evidence, incomplete parameters, or unreliable results, you must repair the issue, gather more evidence, call tools again, or clearly state what cannot be completed. - - The final answer is shown to the user only after verification passes. If the system returns Verification feedback, treat it as a real observation and continue revising. - After thinking, when you believe you can answer the user's question, you can generate a final answer directly to the user without generating code and stop the loop. When generating the final answer, you need to follow these specifications: @@ -183,13 +178,3 @@ final_answer: Original task: {{task}} Please provide a clear and concise summary of the work completed so far. - - -verification: - pre_messages: |- - You are a strict verifier for a ReAct agent. Judge reliability only from the task, candidate answer, tool outputs, and observations. Do not output hidden chain-of-thought. - You must output JSON only. - - post_messages: |- - Verify whether the candidate answer covers the user's intent, is grounded in observations, handles tool errors, uses trustworthy citations, and is formatted for users. - Output fields: passed, score, status, failed_criteria, checks, revision_instruction, user_visible_note. diff --git a/backend/prompts/managed_system_prompt_template_zh.yaml b/backend/prompts/managed_system_prompt_template_zh.yaml index da3d53469..291e336fb 100644 --- a/backend/prompts/managed_system_prompt_template_zh.yaml +++ b/backend/prompts/managed_system_prompt_template_zh.yaml @@ -2,7 +2,7 @@ system_prompt: |- ### 基本信息 - 你是{{APP_NAME}}，{{APP_DESCRIPTION}}，用户ID为{{user_id}} + 你是{{APP_NAME}}，{{APP_DESCRIPTION}}，现在是{{time|default('当前时间')}}，用户ID为{{user_id}} {%- if memory_list and memory_list|length > 0 %} ### 上下文记忆 @@ -130,11 +130,6 @@ system_prompt: |- - 注意运行的代码不会被用户看到，所以如果用户需要看到代码，你需要使用'代码'表达展示代码。 - **重要**：代码执行后，系统会返回 "Observation:" 标记的内容（这是真实的执行结果）。请基于这些真实结果继续下一步思考，**不要在代码执行前自行编造观察结果**。 - 3. 自验证： - - 关键事件（工具调用、检索结果、代码执行、准备最终回答）后，系统会进行显式自验证。 - - 如果自验证提示存在错误、证据不足、参数不完整或结果不可靠，必须优先修正、补充证据、重新调用工具，或清晰说明无法完成的部分。 - - 最终回答只有在自验证通过后才会展示给用户；如果系统返回 Verification feedback，请把它视为真实观察结果继续修正，不要忽略。 - 在思考结束后，当你认为可以回答用户问题，那么可以不生成代码，直接生成最终回答给到用户并停止循环。 生成最终回答时，你需要遵循以下规范： @@ -276,13 +271,3 @@ final_answer: 原始任务：{{task}} 请对迄今为止完成的工作进行清晰、简洁的总结。 - - -verification: - pre_messages: |- - 你是 ReAct 智能体的严格验证器。请仅根据任务、候选答案、工具输出和观察结果判断答案是否可靠，不要输出隐藏思维链。 - 你必须只输出 JSON。 - - post_messages: |- - 请验证候选答案是否覆盖用户意图、是否有观察结果支撑、是否处理了工具错误、引用是否可信、格式是否适合展示。 - 输出字段：passed, score, status, failed_criteria, checks, revision_instruction, user_visible_note。 diff --git a/backend/prompts/manager_system_prompt_template_en.yaml b/backend/prompts/manager_system_prompt_template_en.yaml index d44ed9a71..8ce58db29 100644 --- a/backend/prompts/manager_system_prompt_template_en.yaml +++ b/backend/prompts/manager_system_prompt_template_en.yaml @@ -1,6 +1,6 @@ system_prompt: |- ### Basic Information - You are {{APP_NAME}}, {{APP_DESCRIPTION}} + You are {{APP_NAME}}, {{APP_DESCRIPTION}}, it is {{time|default('current time')}} now {%- if memory_list and memory_list|length > 0 %} ### Contextual Memory @@ -67,11 +67,6 @@ system_prompt: |- - Note that executed code is not visible to users. If users need to see the code, use 'code' for displaying code. - **IMPORTANT**: After code execution, the system will return content with "Observation:" marker (this is the real execution result). Please continue your next thinking based on these real results. **Do NOT fabricate observation results before code execution.** - 3. Self-verification: - - After critical events (tool calls, retrieval results, code execution, agent handoffs, and final-answer preparation), the system may run explicit verification. - - If verification reports errors, insufficient evidence, incomplete parameters, or unreliable results, you must repair the issue, gather more evidence, call tools again, or clearly state what cannot be completed. - - The final answer is shown to the user only after verification passes. If the system returns Verification feedback, treat it as a real observation and continue revising. - After thinking, when you believe you can answer the user's question, you can generate a final answer directly to the user without generating code and stop the loop. When generating the final answer, you need to follow these specifications: @@ -227,13 +222,3 @@ final_answer: Original task: {{task}} Please provide a clear and concise summary of the work completed so far. - - -verification: - pre_messages: |- - You are a strict verifier for a ReAct agent. Judge reliability only from the task, candidate answer, tool outputs, and observations. Do not output hidden chain-of-thought. - You must output JSON only. - - post_messages: |- - Verify whether the candidate answer covers the user's intent, is grounded in observations, handles tool errors, uses trustworthy citations, and is formatted for users. - Output fields: passed, score, status, failed_criteria, checks, revision_instruction, user_visible_note. diff --git a/backend/prompts/manager_system_prompt_template_zh.yaml b/backend/prompts/manager_system_prompt_template_zh.yaml index a49ced82d..fc4eb7c0c 100644 --- a/backend/prompts/manager_system_prompt_template_zh.yaml +++ b/backend/prompts/manager_system_prompt_template_zh.yaml @@ -1,6 +1,6 @@ system_prompt: |- ### 基本信息 - 你是{{APP_NAME}}，{{APP_DESCRIPTION}}，用户ID为{{user_id}} + 你是{{APP_NAME}}，{{APP_DESCRIPTION}}，现在是{{time|default('当前时间')}}，用户ID为{{user_id}} {%- if memory_list and memory_list|length > 0 %} ### 上下文记忆 @@ -130,11 +130,6 @@ system_prompt: |- - 注意运行的代码不会被用户看到，所以如果用户需要看到代码，你需要使用'代码'表达展示代码。 - **重要**：代码执行后，系统会返回 "Observation:" 标记的内容（这是真实的执行结果）。请基于这些真实结果继续下一步思考，**不要在代码执行前自行编造观察结果**。 - 3. 自验证： - - 关键事件（工具调用、检索结果、代码执行、助手返回、准备最终回答）后，系统会进行显式自验证。 - - 如果自验证提示存在错误、证据不足、参数不完整或结果不可靠，必须优先修正、补充证据、重新调用工具，或清晰说明无法完成的部分。 - - 最终回答只有在自验证通过后才会展示给用户；如果系统返回 Verification feedback，请把它视为真实观察结果继续修正，不要忽略。 - 在思考结束后，当你认为可以回答用户问题，那么可以不生成代码，直接生成最终回答给到用户并停止循环。 生成最终回答时，你需要遵循以下规范： @@ -304,13 +299,3 @@ final_answer: 原始任务：{{task}} 请对迄今为止完成的工作进行清晰、简洁的总结。 - - -verification: - pre_messages: |- - 你是 ReAct 智能体的严格验证器。请仅根据任务、候选答案、工具输出和观察结果判断答案是否可靠，不要输出隐藏思维链。 - 你必须只输出 JSON。 - - post_messages: |- - 请验证候选答案是否覆盖用户意图、是否有观察结果支撑、是否处理了工具错误、引用是否可信、格式是否适合展示。 - 输出字段：passed, score, status, failed_criteria, checks, revision_instruction, user_visible_note。 diff --git a/backend/prompts/utils/greeting_generate_en.yaml b/backend/prompts/utils/greeting_generate_en.yaml deleted file mode 100644 index 31ea75632..000000000 --- a/backend/prompts/utils/greeting_generate_en.yaml +++ /dev/null @@ -1,54 +0,0 @@ -GREETING_SYSTEM_PROMPT: |- - ### You are an expert in generating agent greetings and example questions. You help users create engaging greetings and practical example questions for starting conversations with agents. - You are building an Agent application. The input includes: agent name, duty description, business description, and existing examples. - Generate a concise greeting and 3-5 example questions that help users quickly start a conversation with the agent. - The greeting should reflect the agent's positioning and capabilities. - - ### Requirements: - 1. The greeting should be concise and friendly, 1-2 sentences, introducing the agent's identity and core capabilities. Don't make it too long or too formal. - 2. Example questions should be specific and practical, representing questions users might actually ask, showcasing the agent's core features. - 3. If existing examples contain user query scenarios, prioritize extracting short user questions from them, keeping semantics consistent but simplified to natural conversational form. - 4. Provide 3-5 example questions, each with a clear use case. - 5. You MUST output strictly in JSON format, do not output any other content or formatting. - - ### Output format: - ```json - { - "greeting_message": "greeting content", - "example_questions": ["example question 1", "example question 2", "example question 3"] - } - ``` - - ### Examples: - Example 1 (Travel Planning Assistant, existing examples contain "Help me plan a trip from Shanghai to Beijing" etc.): - ```json - { - "greeting_message": "Hello! I'm your travel planning assistant, I can help you plan trips, recommend attractions, and arrange travel routes.", - "example_questions": ["Help me plan a 3-day trip from Shanghai to Beijing", "Recommend some family-friendly attractions", "What's fun to do in Hangzhou tomorrow?"] - } - ``` - - Example 2 (Data Analysis Assistant): - ```json - { - "greeting_message": "Hello! I'm a data analysis assistant, I can help you process and analyze data, provide visual reports and insights.", - "example_questions": ["Help me analyze trends in this sales data", "Generate a quarterly performance comparison report", "Which products have the highest profit margins?"] - } - ``` - -USER_PROMPT: |- - ### Agent Name: - {{display_name}} - - ### Agent Duty Description: - {{duty_description}} - - ### Business Description: - {{business_description}} - - {% if few_shots %} - ### Existing Examples (extract user query scenarios from these as example questions): - {{few_shots}} - {% endif %} - - Please generate the greeting and example questions based on the above information. Output strictly in JSON format. \ No newline at end of file diff --git a/backend/prompts/utils/greeting_generate_zh.yaml b/backend/prompts/utils/greeting_generate_zh.yaml deleted file mode 100644 index 34b8d85d3..000000000 --- a/backend/prompts/utils/greeting_generate_zh.yaml +++ /dev/null @@ -1,53 +0,0 @@ -GREETING_SYSTEM_PROMPT: |- - ### 你是【智能体开场白和示例问题生成专家】，用于帮助用户创建高效、吸引人的智能体开场白和示例问题。 - 现在正在构建一个Agent应用，用户的输入包含：智能体名称、职责描述、业务描述、已有示例。 - 请根据智能体的定位和职责，生成一个简短的开场白和3~5个示例问题，帮助用户快速开始与智能体的对话。 - - ### 要求： - 1.开场白要简洁友好，1-2句话即可，介绍智能体的身份和核心能力，不要过长或过于正式。 - 2.示例问题要具体、实用，是用户真实可能提出的问题，体现智能体的核心功能。 - 3.如果已有示例中包含用户的提问场景，请优先从中提炼简短的用户问题作为示例问题，保持语义一致但简化为自然对话形式。 - 4.示例问题数量为3~5个，每个问题要有明确的使用场景。 - 5.必须严格按照JSON格式输出，不要输出任何其他内容或格式。 - - ### 输出格式： - ```json - { - "greeting_message": "开场白内容", - "example_questions": ["示例问题1", "示例问题2", "示例问题3"] - } - ``` - - ### 参考示例： - 示例1（旅行规划助手，已有示例包含"帮我规划明天从上海出发去北京的行程"等场景）： - ```json - { - "greeting_message": "你好！我是你的旅行规划助手，可以帮你规划行程、推荐景点和安排出行路线。", - "example_questions": ["帮我规划一个从上海到北京的三日旅行", "推荐一些适合家庭出游的景点", "明天去杭州有什么好玩的地方？"] - } - ``` - - 示例2（数据分析助手）： - ```json - { - "greeting_message": "你好！我是数据分析助手，可以帮你处理和分析各种数据，提供可视化报告和洞察。", - "example_questions": ["帮我分析这组销售数据的趋势", "生成一份季度业绩对比报告", "哪些产品的利润率最高？"] - } - ``` - -USER_PROMPT: |- - ### 智能体名称： - {{display_name}} - - ### 智能体职责描述： - {{duty_description}} - - ### 业务描述： - {{business_description}} - - {% if few_shots %} - ### 已有示例（请从中提炼用户提问场景作为示例问题）： - {{few_shots}} - {% endif %} - - 请根据以上信息生成开场白和示例问题。严格按JSON格式输出。 \ No newline at end of file diff --git a/backend/pyproject.toml b/backend/pyproject.toml index b8f51dd4c..dff0e8693 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "backend" version = "0.1.0" -requires-python = ">=3.11,<3.12" +requires-python = "==3.10.*" dependencies = [ "aiofiles>=0.8.0", "uvicorn>=0.34.0", @@ -11,7 +11,7 @@ dependencies = [ "aiohttp>=3.8.0", "authlib>=1.3.0", "cryptography>=42.0.0", - "psycopg2-binary>=2.9.9", + "psycopg2-binary==2.9.10", "PyJWT>=2.8.0", "sqlalchemy~=2.0.37", "greenlet<3.5.0", @@ -21,14 +21,10 @@ dependencies = [ "jsonref>=1.1.0", "ruamel-yaml==0.19.1", "redis>=5.0.0", - "fastmcp>=2.14.2,<3.0", + "fastmcp==2.12.0", "langchain>=0.3.26", "scikit-learn>=1.0.0", "numpy>=1.24.0", - "defusedxml>=0.7.1", - "openjiuwen>=0.1.0", - "pydantic-settings>=2.0.0", - "python-docx>=1.1.0", ] [project.optional-dependencies] @@ -38,7 +34,7 @@ data-process = [ "flower>=2.0.1", "nest_asyncio>=1.5.6", "unstructured[csv,docx,pdf,pptx,xlsx,md]==0.18.14", - "huggingface_hub>=0.30.0,<1.0" + "huggingface_hub>=0.19.0,<0.21.0" ] test = [ "pytest", diff --git a/backend/services/agent_repository_service.py b/backend/services/agent_repository_service.py deleted file mode 100644 index 87649bcd1..000000000 --- a/backend/services/agent_repository_service.py +++ /dev/null @@ -1,306 +0,0 @@ -import logging -from typing import Any, Dict, Optional - -from consts.const import ASSET_OWNER_TENANT_ID -from consts.model import AgentRepositorySnapshot -from database.agent_db import search_agent_info_by_agent_id -from database.agent_version_db import search_version_by_version_no -from database.agent_repository_db import ( - STATUS_PENDING_REVIEW, - VALID_REPOSITORY_STATUSES, - get_agent_repository_by_agent_id, - get_agent_repository_by_id, - insert_agent_repository_record, - list_agent_repository_summaries, - update_agent_repository_by_id, - update_agent_repository_status_by_id, -) -from services.agent_service import ( - collect_skill_zip_entries, - export_agent_dict_for_repository_impl, - import_agent_impl, - import_agent_with_skills_impl, -) - -logger = logging.getLogger("agent_repository_service") - -_UPDATE_SNAPSHOT_FIELDS = ( - "display_name", - "description", - "author", - "category_id", - "tags", - "tool_count", - "version_label", - "source_version_no", - "agent_info_json", - "status", -) - - -def _to_summary_item(record: Dict[str, Any]) -> Dict[str, Any]: - """Map a DB record to a lightweight marketplace summary item.""" - return { - "agent_repository_id": record.get("agent_repository_id"), - "author": record.get("author"), - "name": record.get("name"), - "display_name": record.get("display_name"), - "description": record.get("description"), - "status": record.get("status"), - } - - -def list_agent_repository_listings_impl( - *, - status: Optional[str] = None, -) -> Dict[str, Any]: - """List all repository listings with optional status filter.""" - if status is not None and status not in VALID_REPOSITORY_STATUSES: - raise ValueError( - f"Invalid status '{status}'; must be one of: " - f"{', '.join(sorted(VALID_REPOSITORY_STATUSES))}" - ) - records = list_agent_repository_summaries(status=status) - return {"items": [_to_summary_item(record) for record in records]} - - -def update_agent_repository_status_impl( - *, - agent_repository_id: int, - status: str, - user_id: str, -) -> Dict[str, Any]: - """Update a repository listing status by primary key.""" - if status not in VALID_REPOSITORY_STATUSES: - raise ValueError( - f"Invalid status '{status}'; must be one of: " - f"{', '.join(sorted(VALID_REPOSITORY_STATUSES))}" - ) - - record = get_agent_repository_by_id(agent_repository_id) - if not record: - raise ValueError("Repository listing not found") - - rows_affected = update_agent_repository_status_by_id( - repository_id=agent_repository_id, - status=status, - user_id=user_id, - ) - if rows_affected == 0: - raise ValueError("Repository listing not found") - - updated = get_agent_repository_by_id(agent_repository_id) - if not updated: - raise ValueError("Failed to load repository listing after update") - return _to_summary_item(updated) - - -def _to_list_item(record: Dict[str, Any]) -> Dict[str, Any]: - """Map a DB record to a marketplace list item (without heavy JSON blobs).""" - return { - "id": record.get("agent_repository_id"), - "agent_repository_id": record.get("agent_repository_id"), - "agent_id": record.get("agent_id"), - "name": record.get("name"), - "display_name": record.get("display_name"), - "description": record.get("description"), - "author": record.get("author"), - "category_id": record.get("category_id"), - "tags": record.get("tags") or [], - "tool_count": record.get("tool_count"), - "version_label": record.get("version_label"), - "status": record.get("status"), - "source_version_no": record.get("source_version_no"), - "publisher_tenant_id": record.get("publisher_tenant_id"), - "created_at": record.get("create_time"), - "updated_at": record.get("update_time"), - } - - -def _to_detail_item( - record: Dict[str, Any], - *, - include_bundles: bool = True, - is_updated: Optional[bool] = None, -) -> Dict[str, Any]: - """Map a DB record to a marketplace detail payload.""" - detail = _to_list_item(record) - if include_bundles: - detail["agent_info_json"] = record.get("agent_info_json") - if is_updated is not None: - detail["is_updated"] = is_updated - return detail - - -def _validate_create_payload(repository_data: Dict[str, Any]) -> None: - """Validate required fields before inserting a repository listing.""" - required_fields = ( - "agent_id", - "source_version_no", - "name", - "agent_info_json", - ) - missing = [ - field for field in required_fields - if field not in repository_data or repository_data[field] is None - ] - if missing: - raise ValueError(f"Missing required repository fields: {', '.join(missing)}") - if not repository_data.get("name"): - raise ValueError("name must be a non-empty string") - - agent_info_json = repository_data.get("agent_info_json") - if not isinstance(agent_info_json, dict): - raise ValueError("agent_info_json must be a JSON object") - for key in ("agent_id", "agent_info", "mcp_info"): - if key not in agent_info_json: - raise ValueError(f"agent_info_json must contain '{key}'") - - -def _validate_agent_info_json_shareable(agent_info_json: dict) -> None: - """Reject marketplace share when any agent in the tree belongs to ASSET_OWNER tenant.""" - agent_info_map = agent_info_json.get("agent_info") - if not isinstance(agent_info_map, dict): - return - for entry in agent_info_map.values(): - if not isinstance(entry, dict): - continue - if entry.get("tenant_id") == ASSET_OWNER_TENANT_ID: - raise ValueError("租户管理员智能体无法共享") - - -async def _build_agent_info_json( - agent_id: int, - tenant_id: str, - user_id: str, - version_no: int, -) -> dict: - """Build marketplace snapshot JSON via the agent export pipeline.""" - export_dict = await export_agent_dict_for_repository_impl( - agent_id=agent_id, - tenant_id=tenant_id, - user_id=user_id, - version_no=version_no, - ) - skills = collect_skill_zip_entries( - agent_id=agent_id, - tenant_id=tenant_id, - version_no=version_no, - ) - snapshot = AgentRepositorySnapshot( - **export_dict, - skills=skills or None, - ) - return snapshot.model_dump() - - -async def _build_repository_data_from_agent( - agent_id: int, - tenant_id: str, - user_id: str, - version_no: int, -) -> Dict[str, Any]: - """Build a repository upsert payload from a published agent version snapshot.""" - agent_info = search_agent_info_by_agent_id(agent_id, tenant_id, version_no) - agent_info_json = await _build_agent_info_json( - agent_id=agent_id, - tenant_id=tenant_id, - user_id=user_id, - version_no=version_no, - ) - _validate_agent_info_json_shareable(agent_info_json) - - version_meta = search_version_by_version_no(agent_id, tenant_id, version_no) - version_label = ( - version_meta.get("version_name") - if version_meta and version_meta.get("version_name") - else f"v{version_no}" - ) - - return { - "agent_id": agent_id, - "source_version_no": version_no, - "name": agent_info["name"], - "display_name": agent_info.get("display_name"), - "description": agent_info.get("description"), - "author": agent_info.get("author"), - "version_label": version_label, - "agent_info_json": agent_info_json, - "status": STATUS_PENDING_REVIEW, - } - - -async def create_agent_repository_listing_impl( - agent_id: int, - tenant_id: str, - user_id: str, - version_no: int, -) -> Dict[str, Any]: - """Create or update a repository listing from a published agent version. - - Loads agent metadata and builds agent_info_json via the export pipeline, - then inserts or updates the marketplace table. - - When a listing for the same agent_id already exists, snapshot fields are - updated via update_agent_repository_by_id. - """ - if version_no < 0: - raise ValueError("version_no must be >= 0") - - repository_data = await _build_repository_data_from_agent( - agent_id, tenant_id, user_id, version_no - ) - _validate_create_payload(repository_data) - - existing = get_agent_repository_by_agent_id(agent_id) - if not existing: - repository_id = insert_agent_repository_record( - repository_data=repository_data, - publisher_tenant_id=tenant_id, - publisher_user_id=user_id, - ) - is_updated = False - else: - repository_id = int(existing["agent_repository_id"]) - updates = { - key: repository_data[key] - for key in _UPDATE_SNAPSHOT_FIELDS - if key in repository_data - } - affected = update_agent_repository_by_id( - repository_id=repository_id, - publisher_tenant_id=tenant_id, - user_id=user_id, - updates=updates, - ) - if affected == 0: - raise ValueError("Failed to update repository listing") - is_updated = True - - record = get_agent_repository_by_id(repository_id) - if not record: - raise ValueError("Failed to load repository listing after write") - return _to_detail_item(record, is_updated=is_updated) - - -async def import_agent_from_repository_impl( - agent_repository_id: int, - authorization: str, -) -> Dict[int, int]: - """Import an agent tree from a marketplace repository listing into the current tenant.""" - record = get_agent_repository_by_id(agent_repository_id) - if not record: - raise ValueError("Repository listing not found") - - agent_info_json = record.get("agent_info_json") - if not isinstance(agent_info_json, dict): - raise ValueError("Repository listing has no agent snapshot") - - snapshot = AgentRepositorySnapshot.model_validate(agent_info_json) - if snapshot.skills: - return await import_agent_with_skills_impl( - snapshot, - snapshot.skills, - authorization, - ) - return await import_agent_impl(snapshot, authorization) diff --git a/backend/services/agent_service.py b/backend/services/agent_service.py index 643d1995e..5a340b1d6 100644 --- a/backend/services/agent_service.py +++ b/backend/services/agent_service.py @@ -22,8 +22,7 @@ from utils.prompt_template_utils import normalize_prompt_generate_template_content from consts.const import MEMORY_SEARCH_START_MSG, MEMORY_SEARCH_DONE_MSG, MEMORY_SEARCH_FAIL_MSG, TOOL_TYPE_MAPPING, \ LANGUAGE, MESSAGE_ROLE, MODEL_CONFIG_MAPPING, CAN_EDIT_ALL_USER_ROLES, PERMISSION_EDIT, PERMISSION_READ, PERMISSION_PRIVATE -from consts.exceptions import AppException, MemoryPreparationException, SkillDuplicateError -from consts.error_code import ErrorCode +from consts.exceptions import MemoryPreparationException, SkillDuplicateError from consts.agent_unavailable_reasons import AgentUnavailableReason from consts.model import ( AgentInfoRequest, @@ -46,9 +45,7 @@ delete_related_agent, insert_related_agent, query_all_agent_info_by_tenant_id, - query_sub_agent_relations, query_sub_agents_id_list, - resolve_sub_agent_version_no, search_agent_id_by_agent_name, search_agent_info_by_agent_id, search_blank_sub_agent_by_main_agent_id, @@ -70,10 +67,8 @@ search_tools_for_sub_agent ) from database import skill_db -from database.attachment_db import upload_fileobj from services.skill_service import SkillService -from services.file_management_service import is_allowed_skill_upload_path -from database.agent_version_db import query_version_list, query_current_version_no +from database.agent_version_db import query_version_list from database.group_db import query_group_ids_by_user from database.user_tenant_db import get_user_tenant_by_user_id from database.a2a_agent_db import get_server_agent_ids, query_external_sub_agents @@ -83,7 +78,7 @@ get_prompt_template_summary, ) from utils.str_utils import convert_list_to_string, convert_string_to_list -from services.conversation_management_service import save_conversation_assistant, save_conversation_user, save_skill_files_to_conversation +from services.conversation_management_service import save_conversation_assistant, save_conversation_user from services.memory_config_service import build_memory_context from utils.auth_utils import get_current_user_info, get_user_language from utils.config_utils import tenant_config_manager @@ -102,139 +97,9 @@ SAFE_AGENT_STREAM_ERROR_MESSAGE = "Agent execution failed. Please try again later." -def _extract_json_objects_from_text(text: str) -> list[dict]: - """Extract all JSON objects embedded in a text blob.""" - if not text: - return [] - - decoder = json.JSONDecoder() - results: list[dict] = [] - index = 0 - - while index < len(text): - start_index = text.find("{", index) - if start_index < 0: - break - - try: - payload, end_index = decoder.raw_decode(text, start_index) - except json.JSONDecodeError: - index = start_index + 1 - continue - - if isinstance(payload, dict): - results.append(payload) - index = max(end_index, start_index + 1) - - return results - - -def _extract_skill_file_upload_payloads(content: str) -> list[dict]: - """Extract JSON payloads containing absolute_path from streamed tool output.""" - payloads: list[dict] = [] - for payload in _extract_json_objects_from_text(content): - if payload.get("absolute_path"): - payloads.append(payload) - return payloads - - -def _transform_skill_files_to_standard_format(upload_results: list[dict]) -> list[dict]: - """ - Transform skill file upload results to match the frontend attachment format. - - Skill upload format: - {file_name, absolute_path, object_name, preview_url, url, presigned_url, mime_type, file_size, status} - Frontend format: - {object_name, name, type, size, url, presigned_url, description} - """ - frontend_files = [] - for result in upload_results: - frontend_files.append({ - "object_name": result.get("object_name", ""), - "name": result.get("file_name", result.get("name", "")), - "type": "file", - "size": result.get("file_size", result.get("size", 0)), - "url": result.get("url", ""), - "presigned_url": result.get("presigned_url", result.get("preview_url", "")), - "description": "", - }) - return frontend_files - - -async def _process_skill_file_uploads( - content: str, - user_id: str, - tenant_id: str, -) -> list[dict]: - """Upload generated skill files to storage and return upload metadata.""" - - upload_results: list[dict] = [] - for payload in _extract_skill_file_upload_payloads(content): - absolute_path = str(payload.get("absolute_path") or "").strip() - file_name = str( - payload.get("file_name") - or payload.get("file_path") - or os.path.basename(absolute_path) - ) - mime_type = str(payload.get("mime_type") or payload.get("content_type") or "application/octet-stream") - if not absolute_path: - continue - - if not is_allowed_skill_upload_path(absolute_path): - logger.warning( - "[skill-file] rejected unsafe path absolute_path=%s", - absolute_path, - ) - continue - - if not file_name: - file_name = os.path.basename(absolute_path) - - if not os.path.exists(absolute_path): - continue - - try: - file_size = os.path.getsize(absolute_path) - actual_prefix = f"skill-files/{user_id}" if user_id else "skill-files" - with open(absolute_path, "rb") as file_obj: - upload_result = upload_fileobj( - file_obj=file_obj, - file_name=file_name, - prefix=actual_prefix, - generate_presigned_url=True, - file_size=file_size, - ) - - if upload_result.get("success"): - upload_results.append( - { - "status": "success", - "file_name": file_name, - "absolute_path": absolute_path, - "object_name": upload_result.get("object_name"), - "preview_url": upload_result.get("presigned_url") or upload_result.get("url"), - "url": upload_result.get("url"), - "presigned_url": upload_result.get("presigned_url"), - "mime_type": mime_type, - "file_size": upload_result.get("file_size", file_size), - } - ) - else: - error_message = upload_result.get("error") or "Upload failed" - logger.warning( - "[skill-file] upload failed file_name=%s absolute_path=%s error=%s", - file_name, - absolute_path, - error_message, - ) - except Exception as exc: - logger.exception( - "[skill-file] failed to upload file file_name=%s absolute_path=%s", - file_name, - absolute_path, - ) - - return upload_results +# ------------------------------------------------------------- +# Internal helper functions +# ------------------------------------------------------------- def _safe_agent_stream_error_chunk() -> str: @@ -782,53 +647,23 @@ async def _stream_agent_chunks( agent_run_info, memory_ctx, ): - """Yield SSE chunks from agent_run while persisting messages and cleanup.""" + """Yield SSE chunks from agent_run while persisting messages & cleanup. + + This utility centralizes the common streaming logic used by both + generate_stream_with_memory and generate_stream_no_memory so that the code + is easier to maintain and less error-prone. + """ local_messages = [] captured_final_answer = None - captured_skill_files: dict[str, dict] = {} - skill_file_uploads: list[dict] = [] try: async for chunk in agent_run(agent_run_info): local_messages.append(chunk) + # Try to capture the final answer as it streams by in order to start memory addition try: data = json.loads(chunk) - chunk_type = data.get("type") - if chunk_type == "final_answer": + if data.get("type") == "final_answer": captured_final_answer = data.get("content") - - should_parse_skill_file = chunk_type in {"execution_logs", "parse"} or data.get("role") == "tool-response" - if should_parse_skill_file: - extracted_payload_count = 0 - content_value = data.get("content") - if isinstance(content_value, list): - content_items = content_value - elif content_value: - content_items = [{"type": "text", "text": str(content_value)}] - else: - content_items = [] - - for item in content_items: - if isinstance(item, dict) and item.get("type") == "text": - text_value = item.get("text") - if text_value: - extracted_payloads = _extract_json_objects_from_text(text_value) - for payload in extracted_payloads: - absolute_path = str(payload.get("absolute_path") or "").strip() - if not absolute_path: - continue - if absolute_path in captured_skill_files: - continue - if not os.path.exists(absolute_path): - continue - captured_skill_files[absolute_path] = payload - extracted_payload_count += 1 - if extracted_payload_count: - logger.info( - "[skill-file] captured payloads count=%s current_total=%s", - extracted_payload_count, - len(captured_skill_files), - ) except Exception: pass yield f"data: {chunk}\n\n" @@ -836,6 +671,7 @@ async def _stream_agent_chunks( logger.error("Agent run error: %r", run_exc, exc_info=True) yield _safe_agent_stream_error_chunk() finally: + # Persist assistant messages for non-debug runs if not agent_request.is_debug: save_messages( agent_request, @@ -844,54 +680,11 @@ async def _stream_agent_chunks( tenant_id=tenant_id, user_id=user_id, ) + # Always unregister the run to release resources agent_run_manager.unregister_agent_run( agent_request.conversation_id, user_id) - try: - skill_file_content_local = "\n".join( - json.dumps(payload, ensure_ascii=False) - for payload in captured_skill_files.values() - ) - if skill_file_content_local: - skill_file_uploads = await _process_skill_file_uploads( - content=skill_file_content_local, - user_id=user_id, - tenant_id=tenant_id, - ) - logger.info( - "[skill-file] upload finished conversation=%s result_count=%s results=%s", - agent_request.conversation_id, - len(skill_file_uploads), skill_file_uploads - ) - if skill_file_uploads: - # Keep original format for real-time SSE display - skill_files_payload = json.dumps( - {"skill_file_uploads": skill_file_uploads}, - ensure_ascii=False, - ) - try: - yield f"data: {json.dumps({'type': 'skill_files', 'content': skill_files_payload}, ensure_ascii=False)}\n\n" - except RuntimeError: - # Stream is closing (e.g., client disconnect). Avoid raising during generator teardown. - pass - # Persist skill file uploads to the conversation history so they - # appear in subsequent GET /conversation/{id} calls. - # Transform to frontend attachment format (object_name, name, type, size, etc.) - try: - frontend_files = _transform_skill_files_to_standard_format(skill_file_uploads) - save_skill_files_to_conversation( - conversation_id=agent_request.conversation_id, - skill_file_uploads=frontend_files, - user_id=user_id, - ) - except Exception: - logger.exception( - "[skill-file] failed to persist skill file uploads to conversation=%s", - agent_request.conversation_id, - ) - except Exception: - logger.exception("Failed to process skill file uploads") - + # Schedule memory addition in background to avoid blocking SSE termination async def _add_memory_background(): try: # Skip if memory recording is disabled @@ -986,13 +779,14 @@ async def get_agent_info_impl(agent_id: int, tenant_id: str, version_no: int = 0 user_role = str(user_tenant_record.get("user_role") or "").upper() can_edit_all = user_role in CAN_EDIT_ALL_USER_ROLES - # Permission logic (same as agent list, including ASSET_OWNER read-only override) - agent_info["permission"] = resolve_agent_list_permission( - user_role=user_role, - agent=agent_info, - user_id=user_id, - can_edit_all=can_edit_all, - ) + # Permission logic (same as agent list): + # - If creator or can_edit_all: PERMISSION_EDIT + # - Otherwise: use ingroup_permission, default to PERMISSION_READ if None + if can_edit_all or str(agent_info.get("created_by")) == str(user_id): + agent_info["permission"] = PERMISSION_EDIT + else: + ingroup_permission = agent_info.get("ingroup_permission") + agent_info["permission"] = ingroup_permission if ingroup_permission is not None else PERMISSION_READ except Exception as e: logger.warning(f"Failed to calculate agent permission: {str(e)}") @@ -1068,12 +862,6 @@ async def get_agent_info_impl(agent_id: int, tenant_id: str, version_no: int = 0 agent_info["is_available"] = is_available agent_info["unavailable_reasons"] = unavailable_reasons - # Set current_version_no from draft record (version_no=0) - # This ensures the returned data always has the current published version info - if version_no > 0: - draft_version_no = query_current_version_no(agent_id, tenant_id) - agent_info["current_version_no"] = draft_version_no - return agent_info @@ -1118,10 +906,6 @@ async def get_creating_sub_agent_info_impl(authorization: str = Header(None)): async def update_agent_info_impl(request: AgentInfoRequest, authorization: str = Header(None)): user_id, tenant_id, _ = get_current_user_info(authorization) - - if request.example_questions is not None and len(request.example_questions) > 6: - raise AppException(ErrorCode.COMMON_PARAMETER_INVALID, "example_questions cannot exceed 6 items") - prompt_template_id, prompt_template_name = get_prompt_template_summary( template_id=request.prompt_template_id, tenant_id=tenant_id, @@ -1148,12 +932,9 @@ async def update_agent_info_impl(request: AgentInfoRequest, authorization: str = "prompt_template_name": prompt_template_name, "max_steps": request.max_steps, "provide_run_summary": request.provide_run_summary, - "verification_config": request.verification_config, "duty_prompt": request.duty_prompt, "constraint_prompt": request.constraint_prompt, "few_shots_prompt": request.few_shots_prompt, - "greeting_message": request.greeting_message, - "example_questions": request.example_questions, "enabled": request.enabled if request.enabled is not None else True, "group_ids": convert_list_to_string(request.group_ids) if request.group_ids else user_group_ids, "ingroup_permission": request.ingroup_permission @@ -1421,216 +1202,76 @@ async def clear_agent_memory(agent_id: int, tenant_id: str, user_id: str): # Silently fail to maintain agent deletion process -async def _export_agent_dict_core( - root_agent_id: int, - tenant_id: str, - user_id: str, - version_no: int = 0, -) -> dict: - """Build ExportAndImportDataFormat dict for an agent tree at the given version.""" +async def export_agent_impl(agent_id: int, authorization: str = Header(None)) -> str: + """ + Export the configuration information of the specified agent and all its sub-agents. + + Args: + agent_id (int): The ID of the agent to export. + authorization (str): User authentication information, obtained from the Header. + + Returns: + str: A formatted JSON string containing the configuration information of the agent and all its sub-agents. + + Data Structure Example: + model.py ExportAndImportDataFormat + + Note: + This function recursively finds all managed sub-agents and exports the detailed configuration of each agent (including tools, prompts, etc.) as a dictionary, and finally returns it as a formatted JSON string for frontend download and backup. + """ + + user_id, tenant_id, _ = get_current_user_info(authorization) + export_agent_dict = {} - search_list: deque = deque([(root_agent_id, version_no)]) - visited: set = set() + search_list = deque([agent_id]) + agent_id_set = set() mcp_info_set = set() - while search_list: - current_agent_id, current_version_no = search_list.popleft() - visit_key = (current_agent_id, current_version_no) - if visit_key in visited: + while len(search_list): + left_ele = search_list.popleft() + if left_ele in agent_id_set: continue - visited.add(visit_key) - agent_info = await export_agent_by_agent_id( - agent_id=current_agent_id, - tenant_id=tenant_id, - user_id=user_id, - version_no=current_version_no, - ) + agent_id_set.add(left_ele) + agent_info = await export_agent_by_agent_id(agent_id=left_ele, tenant_id=tenant_id, user_id=user_id) + # collect mcp name for tool in agent_info.tools: if tool.source == "mcp" and tool.usage: mcp_info_set.add(tool.usage) - relations = query_sub_agent_relations( - main_agent_id=current_agent_id, - tenant_id=tenant_id, - version_no=current_version_no, - ) - for rel in relations: - child_id = rel["selected_agent_id"] - child_version = resolve_sub_agent_version_no( - child_id, - rel.get("selected_agent_version_no"), - tenant_id, - ) - search_list.append((child_id, child_version)) - + search_list.extend(agent_info.managed_agents) export_agent_dict[str(agent_info.agent_id)] = agent_info + # convert mcp info to MCPInfo list mcp_info_list = [] for mcp_server_name in mcp_info_set: + # get mcp url by mcp_server_name and tenant_id mcp_url = get_mcp_server_by_name_and_tenant(mcp_server_name, tenant_id) mcp_info_list.append( MCPInfo(mcp_server_name=mcp_server_name, mcp_url=mcp_url)) export_data = ExportAndImportDataFormat( - agent_id=root_agent_id, - agent_info=export_agent_dict, - mcp_info=mcp_info_list, - ) - return export_data.model_dump() + agent_id=agent_id, agent_info=export_agent_dict, mcp_info=mcp_info_list) + return json.dumps(export_data.model_dump()) -async def export_agent_dict_impl( - agent_id: int, - authorization: str = Header(None), - version_no: int = 0, -) -> dict: +async def export_agent_by_agent_id(agent_id: int, tenant_id: str, user_id: str) -> ExportAndImportAgentInfo: """ - Export the configuration information of the specified agent and all its sub-agents. - - Args: - agent_id (int): The ID of the agent to export. - authorization (str): User authentication information, obtained from the Header. - version_no (int): Version to export. Default 0 = draft. - - Returns: - dict: ExportAndImportDataFormat as a plain dict (via model_dump). + Export a single agent's information based on agent_id """ - user_id, tenant_id, _ = get_current_user_info(authorization) - return await _export_agent_dict_core( - root_agent_id=agent_id, - tenant_id=tenant_id, - user_id=user_id, - version_no=version_no, - ) - - -async def export_agent_dict_for_repository_impl( - agent_id: int, - tenant_id: str, - user_id: str, - version_no: int, -) -> dict: - """Export agent tree for marketplace repository storage (no HTTP auth header).""" - return await _export_agent_dict_core( - root_agent_id=agent_id, - tenant_id=tenant_id, - user_id=user_id, - version_no=version_no, - ) - - -async def export_agent_impl( - agent_id: int, - authorization: str = Header(None), - version_no: int = 0, -) -> str: - """Serialize export_agent_dict_impl output to a JSON string for download or ZIP embedding.""" - agent_dict = await export_agent_dict_impl( - agent_id, authorization, version_no=version_no - ) - return json.dumps(agent_dict) - - -def _collect_skill_names_from_tree( - agent_id: int, - tenant_id: str, - version_no: int, - visited: Optional[set] = None, -) -> List[str]: - """Collect unique skill names from an agent tree at the given version.""" - if visited is None: - visited = set() - - skill_names: List[str] = [] - seen_names: set = set() - - def _walk(current_agent_id: int, current_version_no: int) -> None: - visit_key = (current_agent_id, current_version_no) - if visit_key in visited: - return - visited.add(visit_key) - - skill_instances = skill_db.query_skill_instances_by_agent_id( - agent_id=current_agent_id, - tenant_id=tenant_id, - version_no=current_version_no, - ) - for inst in skill_instances: - skill_id = inst.get("skill_id") - skill = skill_db.get_skill_by_id(skill_id, tenant_id) - if skill: - name = skill.get("name") - if name and name not in seen_names: - seen_names.add(name) - skill_names.append(name) - - relations = query_sub_agent_relations( - main_agent_id=current_agent_id, - tenant_id=tenant_id, - version_no=current_version_no, - ) - for rel in relations: - child_id = rel["selected_agent_id"] - child_version = resolve_sub_agent_version_no( - child_id, - rel.get("selected_agent_version_no"), - tenant_id, - ) - _walk(child_id, child_version) - - _walk(agent_id, version_no) - return skill_names - - -def collect_skill_zip_entries( - agent_id: int, - tenant_id: str, - version_no: int = 0, -) -> List[SkillZipEntry]: - """Export skill ZIP payloads for all skills in an agent tree.""" - skill_names = _collect_skill_names_from_tree(agent_id, tenant_id, version_no) - if not skill_names: - return [] - - skill_service = SkillService(tenant_id=tenant_id) - exported = skill_service.export_skills_by_names(skill_names, tenant_id) - return [ - SkillZipEntry( - skill_name=entry["skill_name"], - skill_zip_base64=entry["skill_zip_base64"], - ) - for entry in exported - ] - - -async def export_agent_by_agent_id( - agent_id: int, - tenant_id: str, - user_id: str, - version_no: int = 0, -) -> ExportAndImportAgentInfo: - """Export a single agent's information based on agent_id and version_no.""" agent_info = search_agent_info_by_agent_id( - agent_id=agent_id, tenant_id=tenant_id, version_no=version_no - ) + agent_id=agent_id, tenant_id=tenant_id) agent_relation_in_db = query_sub_agents_id_list( - main_agent_id=agent_id, tenant_id=tenant_id, version_no=version_no - ) - tool_list = await create_tool_config_list( - agent_id=agent_id, - tenant_id=tenant_id, - user_id=user_id, - version_no=version_no, - ) + main_agent_id=agent_id, tenant_id=tenant_id) + tool_list = await create_tool_config_list(agent_id=agent_id, tenant_id=tenant_id, user_id=user_id) # Collect skill names from skill instances skill_names: List[str] = [] try: skill_instances = skill_db.query_skill_instances_by_agent_id( - agent_id=agent_id, tenant_id=tenant_id, version_no=version_no + agent_id=agent_id, tenant_id=tenant_id, version_no=0 ) for inst in skill_instances: skill_id = inst.get("skill_id") @@ -1666,7 +1307,6 @@ async def export_agent_by_agent_id( "display_name") if business_logic_model_info is not None else None agent_info = ExportAndImportAgentInfo(agent_id=agent_id, - tenant_id=agent_info["tenant_id"], name=agent_info["name"], display_name=agent_info["display_name"], description=agent_info["description"], @@ -1674,7 +1314,6 @@ async def export_agent_by_agent_id( author=agent_info.get("author"), max_steps=agent_info["max_steps"], provide_run_summary=agent_info["provide_run_summary"], - verification_config=agent_info.get("verification_config"), duty_prompt=agent_info.get( "duty_prompt"), constraint_prompt=agent_info.get( @@ -1829,7 +1468,6 @@ async def import_agent_by_agent_id( "prompt_template_name": import_agent_info.prompt_template_name or SYSTEM_PROMPT_TEMPLATE_NAME, "max_steps": import_agent_info.max_steps, "provide_run_summary": import_agent_info.provide_run_summary, - "verification_config": getattr(import_agent_info, "verification_config", None), "duty_prompt": import_agent_info.duty_prompt, "constraint_prompt": import_agent_info.constraint_prompt, "few_shots_prompt": import_agent_info.few_shots_prompt, @@ -2197,7 +1835,6 @@ async def prepare_agent_run( is_debug=agent_request.is_debug, override_version_no=agent_request.version_no, override_model_id=agent_request.model_id, - tool_params=agent_request.tool_params, ) # Mount conversation-level reusable ContextManager if enabled @@ -2643,45 +2280,52 @@ def get_sub_agents_recursive(parent_agent_id: int, depth: int = 0, max_depth: in raise ValueError(f"Failed to get agent call relationship: {str(e)}") -async def export_agent_with_skills_impl( - agent_id: int, - authorization: str, - version_no: int = 0, -) -> dict: - """Export an agent, returning a ZIP if it has skill instances, otherwise a plain dict. +async def export_agent_with_skills_impl(agent_id: int, authorization: str) -> dict: + """Export an agent, returning a ZIP if it has skill instances, otherwise plain JSON. The response is either: - A dict with {"_zip": True, "data": bytes, "filename": str} when the agent has skills - - ExportAndImportDataFormat as a plain dict when the agent has no skills + - A plain dict (JSON string) when the agent has no skills """ + from services.skill_service import SkillService + user_id, tenant_id, _ = get_current_user_info(authorization) - skill_zip_entries = collect_skill_zip_entries( - agent_id=agent_id, tenant_id=tenant_id, version_no=version_no + skill_instances = skill_db.query_skill_instances_by_agent_id( + agent_id=agent_id, tenant_id=tenant_id, version_no=0 ) - if not skill_zip_entries: - return await export_agent_dict_impl( - agent_id, authorization, version_no=version_no - ) + if not skill_instances: + return await export_agent_impl(agent_id, authorization) - agent_json_str = await export_agent_impl( - agent_id, authorization, version_no=version_no - ) + skill_names = [] + for inst in skill_instances: + skill_id = inst.get("skill_id") + skill = skill_db.get_skill_by_id(skill_id, tenant_id) + if skill: + skill_names.append(skill.get("name")) + + if not skill_names: + return await export_agent_impl(agent_id, authorization) + + agent_json_str = await export_agent_impl(agent_id, authorization) + + skill_service = SkillService(tenant_id=tenant_id) + skill_zip_entries = skill_service.export_skills_by_names( + skill_names, tenant_id) zip_buffer = io.BytesIO() with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf: zf.writestr("agent.json", agent_json_str) for entry in skill_zip_entries: - skill_zip_bytes = base64.b64decode(entry.skill_zip_base64) - zf.writestr(f"skills/{entry.skill_name}.zip", skill_zip_bytes) + skill_zip_bytes = base64.b64decode(entry["skill_zip_base64"]) + zf.writestr(f"skills/{entry['skill_name']}.zip", skill_zip_bytes) zip_buffer.seek(0) zip_data = zip_buffer.read() agent_info = search_agent_info_by_agent_id( - agent_id=agent_id, tenant_id=tenant_id, version_no=version_no - ) + agent_id=agent_id, tenant_id=tenant_id) agent_name = agent_info.get( "name", "anonymous") if agent_info else "anonymous" diff --git a/backend/services/agent_version_service.py b/backend/services/agent_version_service.py index 8ed6e14d4..d7096727b 100644 --- a/backend/services/agent_version_service.py +++ b/backend/services/agent_version_service.py @@ -49,17 +49,6 @@ def _remove_audit_fields_for_insert(data: dict) -> None: data.pop('delete_flag', None) -def _build_sub_agent_relations(relations: List[dict]) -> List[dict]: - """Map relation snapshots to sub-agent relation payloads for API responses.""" - return [ - { - 'agent_id': r['selected_agent_id'], - 'version_no': r.get('selected_agent_version_no'), - } - for r in relations - ] - - def publish_version_impl( agent_id: int, tenant_id: str, @@ -103,18 +92,11 @@ def publish_version_impl( _remove_audit_fields_for_insert(tool_snapshot) insert_tool_snapshot(tool_snapshot) - # Insert relation snapshots with pinned child agent versions + # Insert relation snapshots for rel in relations_draft: - child_id = rel['selected_agent_id'] - child_version = query_current_version_no(child_id, tenant_id) - if child_version is None: - raise ValueError( - f"Sub-agent {child_id} has no published version; publish the sub-agent first." - ) rel_snapshot = rel.copy() rel_snapshot.pop('version_no', None) rel_snapshot['version_no'] = new_version_no - rel_snapshot['selected_agent_version_no'] = child_version _remove_audit_fields_for_insert(rel_snapshot) insert_relation_snapshot(rel_snapshot) @@ -289,7 +271,6 @@ def get_version_detail_impl( # Extract sub_agent_id_list from relations result['sub_agent_id_list'] = [r['selected_agent_id'] for r in relations_snapshot] - result['sub_agent_relations'] = _build_sub_agent_relations(relations_snapshot) # Get skill instances for this version (from ag_skill_instance_t with version_no) from database import skill_db as skill_db_module @@ -729,7 +710,6 @@ def _get_version_detail_or_draft( # Add tools (only enabled tools) result['tools'] = [t for t in tools_draft if t.get('enabled', True)] result['sub_agent_id_list'] = [r['selected_agent_id'] for r in relations_draft] - result['sub_agent_relations'] = _build_sub_agent_relations(relations_draft) # Get draft skill instances (version_no=0) skills_draft = skill_db_module.query_skill_instances_by_agent_id( @@ -803,11 +783,12 @@ async def list_published_agents_impl( CAN_EDIT_ALL_USER_ROLES, get_user_tenant_by_user_id, query_group_ids_by_user, + PERMISSION_EDIT, + PERMISSION_READ, get_model_by_model_id, check_agent_availability, _apply_duplicate_name_availability_rules, ) - from services.asset_owner_visibility import resolve_agent_list_permission from database.agent_version_db import query_agent_snapshot # Get user role for permission check @@ -877,10 +858,9 @@ async def list_published_agents_impl( # Extract sub_agent_id_list from relations agent_info['sub_agent_id_list'] = [r['selected_agent_id'] for r in relations_snapshot] - agent_info['sub_agent_relations'] = _build_sub_agent_relations(relations_snapshot) - # Add current version info - agent_info['current_version_no'] = current_version_no + # Add published version info + agent_info['published_version_no'] = current_version_no # Check agent availability using the shared function _, unavailable_reasons = check_agent_availability( @@ -913,12 +893,7 @@ async def list_published_agents_impl( model_cache[model_id] = get_model_by_model_id(model_id, tenant_id) model_info = model_cache.get(model_id) - permission = resolve_agent_list_permission( - user_role=user_role, - agent=agent, - user_id=user_id, - can_edit_all=can_edit_all, - ) + permission = PERMISSION_EDIT if can_edit_all or str(agent.get("created_by")) == str(user_id) else PERMISSION_READ simple_agent_list.append({ "agent_id": agent.get("agent_id"), @@ -934,9 +909,7 @@ async def list_published_agents_impl( "is_new": agent.get("is_new", False), "group_ids": agent.get("group_ids", []), "permission": permission, - "current_version_no": agent.get("current_version_no"), - "greeting_message": agent.get("greeting_message"), - "example_questions": agent.get("example_questions"), + "published_version_no": agent.get("published_version_no"), }) return simple_agent_list diff --git a/backend/services/cas_service.py b/backend/services/cas_service.py deleted file mode 100644 index 7db3fce1a..000000000 --- a/backend/services/cas_service.py +++ /dev/null @@ -1,424 +0,0 @@ -import json -import logging -import os -import secrets -import ssl -import urllib.parse -import urllib.request -from xml.etree.ElementTree import Element -from dataclasses import dataclass -from datetime import datetime, timedelta -from typing import Any, Dict, Optional - -import defusedxml.ElementTree as ET -from defusedxml.common import DefusedXmlException - -from consts.const import ( - CAS_CA_BUNDLE, - CAS_CALLBACK_BASE_URL, - CAS_EMAIL_ATTRIBUTE, - CAS_ENABLED, - CAS_LOGIN_MODE, - CAS_LOGOUT_URL, - CAS_RENEW_BEFORE_SECONDS, - CAS_RENEW_TIMEOUT_SECONDS, - CAS_ROLE_ATTRIBUTE, - CAS_ROLE_MAP_JSON, - CAS_SERVER_URL, - CAS_SESSION_MAX_AGE_SECONDS, - CAS_SSL_VERIFY, - CAS_SYNTHETIC_EMAIL_DOMAIN, - CAS_TENANT_ATTRIBUTE, - CAS_USER_ATTRIBUTE, - CAS_VALIDATE_PATH, - DEFAULT_TENANT_ID, - LOCAL_SESSION_MAX_AGE_SECONDS, -) -from database.cas_session_db import ( - create_cas_session, - revoke_cas_session_by_index, - revoke_cas_sessions_by_user_id, -) -from database.oauth_account_db import get_oauth_account_by_provider -from database.user_tenant_db import get_user_tenant_by_user_id, upsert_user_tenant -from services.oauth_service import ( - create_or_update_oauth_account, - find_supabase_user_id_by_email, -) -from services.skill_service import init_skill_list_for_tenant -from services.tool_configuration_service import init_tool_list_for_tenant -from utils.auth_utils import calculate_expires_at, generate_session_jwt, get_supabase_admin_client - -logger = logging.getLogger(__name__) - -CAS_PROVIDER = "cas" -VALID_ROLES = {"SU", "ADMIN", "DEV", "USER"} - - -class CasAuthenticationError(Exception): - pass - - -@dataclass -class CasPrincipal: - cas_user_id: str - email: str - username: str - role: str - tenant_id: str - session_index: str - expires_at: datetime - - -def get_cas_config() -> Dict[str, Any]: - mode = CAS_LOGIN_MODE if CAS_LOGIN_MODE in {"button", "force", "disabled"} else "disabled" - enabled = CAS_ENABLED and bool(CAS_SERVER_URL) - if not enabled: - mode = "disabled" - return { - "enabled": enabled, - "login_mode": mode, - "renew_before_seconds": CAS_RENEW_BEFORE_SECONDS, - "renew_timeout_seconds": CAS_RENEW_TIMEOUT_SECONDS, - "display_name": "CAS", - } - - -def build_login_url(redirect: str = "/") -> str: - _ensure_enabled() - service_url = _build_callback_url("/api/user/cas/callback", {"redirect": _normalize_redirect(redirect)}) - return f"{CAS_SERVER_URL}/login?service={service_url}" - - -def build_renew_url() -> str: - _ensure_enabled() - service_url = _build_callback_url("/api/user/cas/renew_callback", {}) - return f"{CAS_SERVER_URL}/login?service={service_url}&gateway=true" - - -def build_logout_url() -> str: - _ensure_enabled() - configured_logout_url = CAS_LOGOUT_URL.strip() - if not configured_logout_url: - return "" - - parsed_config = urllib.parse.urlsplit(configured_logout_url) - if parsed_config.scheme and parsed_config.netloc: - logout_url = configured_logout_url - else: - logout_url = f"{CAS_SERVER_URL}/{configured_logout_url.lstrip('/')}" - - parsed = urllib.parse.urlsplit(logout_url) - if parsed.query: - return logout_url - - query = f"service={CAS_CALLBACK_BASE_URL}" - return urllib.parse.urlunsplit((parsed.scheme, parsed.netloc, parsed.path, query, parsed.fragment)) - - -async def login_with_ticket(ticket: str, redirect: str = "/") -> Dict[str, Any]: - redirect = _normalize_redirect(redirect) - service_url = _build_callback_url("/api/user/cas/callback", {"redirect": redirect}) - principal = validate_service_ticket(ticket, service_url) - return await _create_project_session(principal, redirect=redirect) - - -async def renew_with_ticket(ticket: str) -> Dict[str, Any]: - service_url = _build_callback_url("/api/user/cas/renew_callback", {}) - principal = validate_service_ticket(ticket, service_url) - return await _create_project_session(principal, redirect="/", renew=True) - - -def validate_service_ticket(ticket: str, service_url: str) -> CasPrincipal: - _ensure_enabled() - if not ticket: - raise CasAuthenticationError("CAS ticket is missing") - - validate_path = CAS_VALIDATE_PATH if CAS_VALIDATE_PATH.startswith("/") else f"/{CAS_VALIDATE_PATH}" - validate_url = f"{CAS_SERVER_URL}{validate_path}" - xml_text = _http_get_text(f"{validate_url}?service={service_url}&ticket={ticket}") - logger.info("CAS serviceValidate response: %s", xml_text) - return parse_service_validate_response(xml_text, fallback_session_index=ticket) - - -def parse_service_validate_response(xml_text: str, fallback_session_index: str = "") -> CasPrincipal: - try: - root = ET.fromstring(xml_text) - except (ET.ParseError, DefusedXmlException) as exc: - raise CasAuthenticationError("Invalid CAS validation response") from exc - - failure = _find_first(root, "authenticationFailure") - if failure is not None: - raise CasAuthenticationError((failure.text or "CAS authentication failed").strip()) - - success = _find_first(root, "authenticationSuccess") - if success is None: - raise CasAuthenticationError("CAS authentication failed") - - user = _get_child_text(success, "user") - attrs_node = _find_first(success, "attributes") - attrs = _extract_attributes(attrs_node) if attrs_node is not None else {} - - cas_user_id = _attribute_or_default(attrs, CAS_USER_ATTRIBUTE, user) or user - if not cas_user_id: - raise CasAuthenticationError("CAS user id is missing") - - email = _attribute_or_default(attrs, CAS_EMAIL_ATTRIBUTE, "") - username = attrs.get("displayName") or attrs.get("name") or cas_user_id - role = _map_role(_attribute_or_default(attrs, CAS_ROLE_ATTRIBUTE, "USER")) - tenant_id = _attribute_or_default(attrs, CAS_TENANT_ATTRIBUTE, DEFAULT_TENANT_ID) or DEFAULT_TENANT_ID - session_index = attrs.get("SessionIndex") or attrs.get("sessionIndex") or fallback_session_index - expires_at = _resolve_expires_at(attrs) - - if not email: - safe_user = "".join(c if c.isalnum() or c in ("-", "_", ".") else "_" for c in cas_user_id) - email = f"{safe_user}@{CAS_SYNTHETIC_EMAIL_DOMAIN}" - - return CasPrincipal( - cas_user_id=str(cas_user_id), - email=str(email).lower(), - username=str(username), - role=role, - tenant_id=str(tenant_id), - session_index=str(session_index or ""), - expires_at=expires_at, - ) - - -def parse_logout_request(logout_request: str) -> Dict[str, str]: - if not logout_request: - return {"cas_user_id": "", "session_index": ""} - try: - root = ET.fromstring(logout_request) - except (ET.ParseError, DefusedXmlException): - logger.warning("Invalid CAS logoutRequest XML") - return {"cas_user_id": "", "session_index": ""} - - session_index = _get_child_text(root, "SessionIndex") - cas_user_id = ( - _get_child_text(root, "NameID") - or _get_child_text(root, "nameID") - or _get_child_text(root, "user") - or _get_child_text(root, "casUserId") - ) - return {"cas_user_id": cas_user_id or "", "session_index": session_index or ""} - - -def revoke_from_logout_request(logout_request: str) -> Dict[str, Any]: - parsed = parse_logout_request(logout_request) - revoked = 0 - if parsed["cas_user_id"]: - revoked = revoke_cas_sessions_by_user_id(parsed["cas_user_id"]) - logger.info( - "CAS SLO revoke by cas_user_id: cas_user_id=%s revoked=%s", - parsed["cas_user_id"], - revoked, - ) - if revoked == 0 and parsed["session_index"]: - revoked = revoke_cas_session_by_index(parsed["session_index"]) - logger.info( - "CAS SLO revoke by session_index: session_index=%s revoked=%s", - parsed["session_index"], - revoked, - ) - if revoked == 0: - logger.warning("CAS SLO did not revoke any session: %s", parsed) - return {"revoked": revoked, **parsed} - - -async def _create_project_session(principal: CasPrincipal, redirect: str = "/", renew: bool = False) -> Dict[str, Any]: - user_id = _resolve_project_user(principal) - existing_tenant = get_user_tenant_by_user_id(user_id) - user_tenant = upsert_user_tenant( - user_id=user_id, - tenant_id=principal.tenant_id, - user_role=principal.role, - user_email=principal.email, - ) - if not existing_tenant: - await init_tool_list_for_tenant(principal.tenant_id, user_id) - await init_skill_list_for_tenant(principal.tenant_id, user_id) - - now = datetime.now() - max_local_expiry = now + timedelta(seconds=LOCAL_SESSION_MAX_AGE_SECONDS) - expires_at_dt = min(principal.expires_at, max_local_expiry) - expires_in_seconds = max(1, int((expires_at_dt - now).total_seconds())) - - session_id = secrets.token_urlsafe(32) - create_cas_session( - session_id=session_id, - user_id=user_id, - cas_user_id=principal.cas_user_id, - cas_session_index=principal.session_index, - expires_at=expires_at_dt, - ) - - jwt_token = generate_session_jwt(user_id, expires_in=expires_in_seconds, session_id=session_id) - - return { - "user": { - "id": str(user_id), - "email": principal.email, - "role": user_tenant.get("user_role", principal.role), - }, - "session": { - "access_token": jwt_token, - "refresh_token": "", - "expires_at": calculate_expires_at(jwt_token), - "expires_in_seconds": expires_in_seconds, - }, - "redirect_url": redirect, - "renew": renew, - } - - -def _resolve_project_user(principal: CasPrincipal) -> str: - existing = get_oauth_account_by_provider(CAS_PROVIDER, principal.cas_user_id) - if existing: - create_or_update_oauth_account( - user_id=existing["user_id"], - provider=CAS_PROVIDER, - provider_user_id=principal.cas_user_id, - email=principal.email, - username=principal.username, - tenant_id=principal.tenant_id, - ) - return existing["user_id"] - - admin_client = get_supabase_admin_client() - if not admin_client: - raise RuntimeError("Supabase admin client not available") - - user_id = find_supabase_user_id_by_email(admin_client, principal.email) - if not user_id: - create_resp = admin_client.auth.admin.create_user( - { - "email": principal.email, - "password": secrets.token_urlsafe(32), - "email_confirm": True, - "user_metadata": { - "full_name": principal.username, - "provider": CAS_PROVIDER, - "cas_user_id": principal.cas_user_id, - }, - } - ) - user_id = create_resp.user.id - - create_or_update_oauth_account( - user_id=user_id, - provider=CAS_PROVIDER, - provider_user_id=principal.cas_user_id, - email=principal.email, - username=principal.username, - tenant_id=principal.tenant_id, - ) - return user_id - - -def _ensure_enabled() -> None: - if not CAS_ENABLED or not CAS_SERVER_URL: - raise CasAuthenticationError("CAS is not configured") - - -def _build_callback_url(path: str, params: Dict[str, str]) -> str: - if not CAS_CALLBACK_BASE_URL: - raise CasAuthenticationError("CAS callback base URL is not configured") - query = _build_callback_query(params) - suffix = f"?{query}" if query else "" - return f"{CAS_CALLBACK_BASE_URL}{path}{suffix}" - - -def _build_callback_query(params: Dict[str, str]) -> str: - return "&".join(f"{key}={value}" for key, value in params.items()) - - -def _normalize_redirect(redirect: str) -> str: - if not redirect or not redirect.startswith("/") or redirect.startswith("//"): - return "/" - return redirect - - -def _build_ssl_context() -> ssl.SSLContext: - if CAS_CA_BUNDLE and os.path.isfile(CAS_CA_BUNDLE): - return ssl.create_default_context(cafile=CAS_CA_BUNDLE) - if not CAS_SSL_VERIFY: - ctx = ssl.create_default_context() - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE - return ctx - return ssl.create_default_context() - - -def _http_get_text(url: str) -> str: - req = urllib.request.Request(url, headers={"Accept": "application/xml,text/xml,*/*"}) - with urllib.request.urlopen(req, timeout=15, context=_build_ssl_context()) as resp: - return resp.read().decode("utf-8") - - -def _local_name(tag: str) -> str: - return tag.rsplit("}", 1)[-1] - - -def _find_first(node: Element, name: str) -> Optional[Element]: - for child in node.iter(): - if _local_name(child.tag) == name: - return child - return None - - -def _get_child_text(node: Element, name: str) -> str: - found = _find_first(node, name) - return (found.text or "").strip() if found is not None else "" - - -def _extract_attributes(attrs_node: Element) -> Dict[str, str]: - attrs: Dict[str, str] = {} - for child in list(attrs_node): - value = (child.text or "").strip() - if value: - attrs[_local_name(child.tag)] = value - return attrs - - -def _attribute_or_default(attrs: Dict[str, str], key: str, default: str) -> str: - if key and key in attrs: - return attrs[key] - return default - - -def _map_role(raw_role: str) -> str: - role = (raw_role or "USER").upper() - try: - role_map = json.loads(CAS_ROLE_MAP_JSON) if CAS_ROLE_MAP_JSON else {} - role = str(role_map.get(raw_role, role_map.get(role, role))).upper() - except Exception: - logger.warning("Invalid CAS_ROLE_MAP_JSON; falling back to raw role") - return role if role in VALID_ROLES else "USER" - - -def _resolve_expires_at(attrs: Dict[str, str]) -> datetime: - for key in ("expiresAt", "expirationDate", "validUntil", "notOnOrAfter"): - value = attrs.get(key) - if not value: - continue - parsed = _parse_datetime(value) - if parsed: - return parsed - return datetime.now() + timedelta(seconds=CAS_SESSION_MAX_AGE_SECONDS) - - -def _parse_datetime(value: str) -> Optional[datetime]: - try: - if value.isdigit(): - timestamp = int(value) - if timestamp > 10_000_000_000: - timestamp = timestamp / 1000 - return datetime.fromtimestamp(timestamp) - normalized = value.replace("Z", "+00:00") - parsed = datetime.fromisoformat(normalized) - if parsed.tzinfo: - parsed = parsed.astimezone().replace(tzinfo=None) - return parsed - except Exception: - return None diff --git a/backend/services/conversation_management_service.py b/backend/services/conversation_management_service.py index 0b7345461..302ec63a8 100644 --- a/backend/services/conversation_management_service.py +++ b/backend/services/conversation_management_service.py @@ -8,7 +8,6 @@ from consts.const import LANGUAGE, MODEL_CONFIG_MAPPING, MESSAGE_ROLE, DEFAULT_EN_TITLE, DEFAULT_ZH_TITLE from consts.model import AgentRequest, ConversationResponse, MessageRequest, MessageUnit -from consts.exceptions import ConversationNotFoundError from database.conversation_db import ( create_conversation, create_conversation_message, @@ -19,14 +18,12 @@ get_conversation, get_conversation_history, get_conversation_list, - get_latest_assistant_message_id, get_message_id_by_index, get_source_images_by_conversation, get_source_images_by_message, get_source_searches_by_conversation, get_source_searches_by_message, rename_conversation, - update_message_minio_files, update_message_opinion ) from nexent.core.utils.observer import MessageObserver, ProcessType @@ -227,7 +224,7 @@ def save_conversation_assistant(request: AgentRequest, messages: List[str], user message_list.append(message) conversation_req = MessageRequest(conversation_id=request.conversation_id, message_idx=user_role_count * 2 + 1, - role=MESSAGE_ROLE["ASSISTANT"], message=message_list, minio_files=None) + role=MESSAGE_ROLE["ASSISTANT"], message=message_list, minio_files=request.minio_files) save_message(conversation_req, user_id=user_id, tenant_id=tenant_id) @@ -299,9 +296,7 @@ def update_conversation_title(conversation_id: int, title: str, user_id: str = N """ success = rename_conversation(conversation_id, title, user_id) if not success: - raise ConversationNotFoundError( - f"Conversation {conversation_id} does not exist or has been deleted" - ) + raise Exception(f"Conversation {conversation_id} does not exist or has been deleted") return success @@ -514,10 +509,6 @@ def get_conversation_history_service(conversation_id: int, user_id: str) -> List 'opinion_flag': msg['opinion_flag'] } - # Add minio_files field (if any, e.g., skill-generated attachments) - if 'minio_files' in msg and msg['minio_files']: - message_item['minio_files'] = msg['minio_files'] - # Add image content (if any) if message_id in image_by_message: message_item['picture'] = image_by_message[message_id] @@ -710,52 +701,3 @@ async def get_message_id_by_index_impl(conversation_id: int, message_index: int) if message_id is None: raise Exception("Message not found.") return message_id - - -def save_skill_files_to_conversation( - conversation_id: int, - skill_file_uploads: List[Dict[str, Any]], - user_id: str, -) -> bool: - """ - Append skill file upload records to the latest assistant message in a conversation. - - This persists generated documents (e.g., DOCX, XLSX created by skills) to the - conversation history so they appear in subsequent GET /conversation/{id} calls. - - Args: - conversation_id: Target conversation ID - skill_file_uploads: List of upload metadata dicts (e.g., from upload_fileobj) - user_id: User ID for ownership validation - - Returns: - bool: True if files were saved, False if no assistant message was found - """ - if not skill_file_uploads: - return False - - try: - message_id = get_latest_assistant_message_id(conversation_id, user_id) - if message_id is None: - logging.warning( - "[skill-file] no assistant message found for conversation=%s, " - "cannot persist skill file uploads", - conversation_id, - ) - return False - - success = update_message_minio_files(message_id, skill_file_uploads) - if success: - logging.info( - "[skill-file] persisted %d file(s) to message_id=%s conversation=%s", - len(skill_file_uploads), - message_id, - conversation_id, - ) - return success - except Exception as exc: - logging.exception( - "[skill-file] failed to persist skill file uploads for conversation=%s", - conversation_id, - ) - return False diff --git a/backend/services/data_process_service.py b/backend/services/data_process_service.py index a7529127c..ae3d35dcd 100644 --- a/backend/services/data_process_service.py +++ b/backend/services/data_process_service.py @@ -15,7 +15,7 @@ import redis import torch from PIL import Image -from celery import states +from celery import states, chain from transformers import CLIPProcessor, CLIPModel from nexent.data_process.core import DataProcessCore @@ -25,7 +25,7 @@ from database.attachment_db import delete_file, file_exists, get_file_size_from_minio, get_file_stream, upload_file from utils.file_management_utils import convert_office_to_pdf from data_process.app import app as celery_app -from data_process.tasks import submit_process_forward_chain +from data_process.tasks import process, forward from data_process.utils import get_task_info, get_all_task_ids_from_redis # Limit concurrent LibreOffice processes to avoid resource exhaustion @@ -54,8 +54,7 @@ def __init__(self): self._inspector = None self._inspector_last_time = 0 - # 5 minutes - inspector is expensive to create (ping all workers) - self._inspector_ttl = 300 + self._inspector_ttl = 300 # 5 minutes - inspector is expensive to create (ping all workers) self._inspector_lock = None self._inspector_lock = threading.Lock() @@ -153,8 +152,7 @@ async def get_all_tasks(self, filter: bool = True) -> List[Dict[str, Any]]: def _normalize_runtime_meta(task: Dict[str, Any]) -> Dict[str, Any]: task_name_full = task.get('name', '') or '' - task_name = task_name_full.split( - '.')[-1] if task_name_full else '' + task_name = task_name_full.split('.')[-1] if task_name_full else '' kwargs = task.get('kwargs') or {} if isinstance(kwargs, str): try: @@ -180,43 +178,35 @@ def _normalize_runtime_meta(task: Dict[str, Any]) -> Dict[str, Any]: def get_active(): t = time.time() # Create fresh inspector with short timeout for each call - short_inspector = celery_app.control.inspect( - timeout=short_timeout) + short_inspector = celery_app.control.inspect(timeout=short_timeout) result = short_inspector.active() elapsed = time.time() - t - logger.info( - f"[get_all_tasks] inspector.active() took {elapsed:.3f}s") + logger.info(f"[get_all_tasks] inspector.active() took {elapsed:.3f}s") return result if result else {} def get_reserved(): t = time.time() - short_inspector = celery_app.control.inspect( - timeout=short_timeout) + short_inspector = celery_app.control.inspect(timeout=short_timeout) result = short_inspector.reserved() elapsed = time.time() - t - logger.info( - f"[get_all_tasks] inspector.reserved() took {elapsed:.3f}s") + logger.info(f"[get_all_tasks] inspector.reserved() took {elapsed:.3f}s") return result if result else {} with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: future_active = executor.submit(get_active) future_reserved = executor.submit(get_reserved) - active_tasks_dict = future_active.result( - timeout=short_timeout + 0.5) - reserved_tasks_dict = future_reserved.result( - timeout=short_timeout + 0.5) + active_tasks_dict = future_active.result(timeout=short_timeout + 0.5) + reserved_tasks_dict = future_reserved.result(timeout=short_timeout + 0.5) celery_duration = time.time() - celery_start if celery_duration > 0.5: - logger.warning( - f"[get_all_tasks] Inspector took {celery_duration:.3f}s (expected <0.5s)") + logger.warning(f"[get_all_tasks] Inspector took {celery_duration:.3f}s (expected <0.5s)") if active_tasks_dict: for worker, tasks in active_tasks_dict.items(): for task in tasks: task_id = task.get('id') if task_id: task_ids.add(task_id) - runtime_task_meta[task_id] = _normalize_runtime_meta( - task) + runtime_task_meta[task_id] = _normalize_runtime_meta(task) if reserved_tasks_dict: for worker, tasks in reserved_tasks_dict.items(): for task in tasks: @@ -224,8 +214,7 @@ def get_reserved(): if task_id: task_ids.add(task_id) # Keep active metadata if already present - runtime_task_meta.setdefault( - task_id, _normalize_runtime_meta(task)) + runtime_task_meta.setdefault(task_id, _normalize_runtime_meta(task)) # Get task IDs from Redis backend (covers completed/failed tasks within expiry) try: @@ -252,14 +241,11 @@ def get_reserved(): if not task_info.get('task_name') and runtime_meta.get('task_name'): task_info['task_name'] = runtime_meta.get('task_name') if not task_info.get('index_name') and runtime_meta.get('index_name'): - task_info['index_name'] = runtime_meta.get( - 'index_name') + task_info['index_name'] = runtime_meta.get('index_name') if not task_info.get('path_or_url') and runtime_meta.get('path_or_url'): - task_info['path_or_url'] = runtime_meta.get( - 'path_or_url') + task_info['path_or_url'] = runtime_meta.get('path_or_url') if not task_info.get('original_filename') and runtime_meta.get('original_filename'): - task_info['original_filename'] = runtime_meta.get( - 'original_filename') + task_info['original_filename'] = runtime_meta.get('original_filename') if filter and not (task_info.get('index_name') and task_info.get('task_name')): # Keep user-visible queued tasks even before worker updates task meta. @@ -552,23 +538,30 @@ async def create_batch_tasks_impl(self, authorization: Optional[str], request: B f"Missing required field 'index_name' in source config: {source_config}") continue - chain_id = submit_process_forward_chain( - source=source, - source_type=source_type, - chunking_strategy=chunking_strategy, - index_name=index_name, - original_filename=original_filename, - authorization=authorization, - embedding_model_id=embedding_model_id, - tenant_id=tenant_id, + # Create and submit a chain: process -> forward + task_chain = chain( + process.s( + source=source, + source_type=source_type, + chunking_strategy=chunking_strategy, + index_name=index_name, + original_filename=original_filename, + embedding_model_id=embedding_model_id, + tenant_id=tenant_id + ).set(queue='process_q'), + forward.s( + index_name=index_name, + source=source, + source_type=source_type, + original_filename=original_filename, + authorization=authorization + ).set(queue='forward_q') ) - if not chain_id: - logger.error( - f"Failed to enqueue process-forward chain for source: {source}") - continue - task_ids.append(chain_id) - logger.debug(f"Created task {chain_id} for source: {source}") + task_result = task_chain.apply_async() + + task_ids.append(task_result.id) + logger.debug(f"Created task {task_result.id} for source: {source}") logger.info( f"Created {len(task_ids)} individual tasks for batch processing") return task_ids @@ -600,7 +593,7 @@ async def process_uploaded_text_file(self, file_content: bytes, filename: str, c f"Processing uploaded file: {filename} using SDK DataProcessCore") data_processor = DataProcessCore() - chunks, _ = data_processor.file_process( + chunks = data_processor.file_process( file_data=file_content, filename=filename, chunking_strategy=chunking_strategy @@ -649,8 +642,7 @@ async def convert_office_to_pdf_impl(self, object_name: str, pdf_object_name: st # Step 1: Download original Office file from MinIO original_stream = get_file_stream(object_name) if original_stream is None: - raise OfficeConversionException( - f"Source file not found in storage: {object_name}") + raise OfficeConversionException(f"Source file not found in storage: {object_name}") original_filename = os.path.basename(object_name) input_path = os.path.join(temp_dir, original_filename) @@ -662,12 +654,10 @@ async def convert_office_to_pdf_impl(self, object_name: str, pdf_object_name: st try: pdf_path = await convert_office_to_pdf(input_path, temp_dir, timeout=30) except Exception as exc: - raise OfficeConversionException( - f"LibreOffice conversion failed: {exc}") from exc + raise OfficeConversionException(f"LibreOffice conversion failed: {exc}") from exc # Step 3: Upload converted PDF to MinIO - result = upload_file(file_path=pdf_path, - object_name=pdf_object_name) + result = upload_file(file_path=pdf_path, object_name=pdf_object_name) if not result.get('success'): raise OfficeConversionException( f"Failed to upload PDF to MinIO: {result.get('error', 'Unknown error')}" @@ -676,16 +666,14 @@ async def convert_office_to_pdf_impl(self, object_name: str, pdf_object_name: st # Step 4: Validate the uploaded PDF (header check + minimum size) remote_size = get_file_size_from_minio(pdf_object_name) if remote_size <= 0: - raise OfficeConversionException( - "PDF validation failed: cannot read remote file size") + raise OfficeConversionException("PDF validation failed: cannot read remote file size") if remote_size < 100: raise OfficeConversionException( f"PDF validation failed: file too small ({remote_size} bytes)" ) remote_stream = get_file_stream(pdf_object_name) if remote_stream is None: - raise OfficeConversionException( - "PDF validation failed: cannot read uploaded file") + raise OfficeConversionException("PDF validation failed: cannot read uploaded file") try: header = remote_stream.read(5) finally: @@ -694,8 +682,7 @@ async def convert_office_to_pdf_impl(self, object_name: str, pdf_object_name: st except Exception: pass if not header.startswith(b'%PDF-'): - raise OfficeConversionException( - "PDF validation failed: invalid PDF header") + raise OfficeConversionException("PDF validation failed: invalid PDF header") except OfficeConversionException: # Clean up any partially-uploaded remote PDF so a future retry starts clean @@ -703,16 +690,14 @@ async def convert_office_to_pdf_impl(self, object_name: str, pdf_object_name: st delete_file(pdf_object_name) raise except Exception as exc: - raise OfficeConversionException( - f"Unexpected error during conversion: {exc}") from exc + raise OfficeConversionException(f"Unexpected error during conversion: {exc}") from exc finally: # Step 5: Clean up local temporary directory if temp_dir and os.path.exists(temp_dir): try: shutil.rmtree(temp_dir) except Exception as cleanup_err: - logger.warning( - f"Failed to cleanup temp dir '{temp_dir}': {cleanup_err}") + logger.warning(f"Failed to cleanup temp dir '{temp_dir}': {cleanup_err}") def convert_celery_states_to_custom(self, process_celery_state: Optional[str], forward_celery_state: Optional[str]) -> str: """Map Celery task states to a custom frontend state string. diff --git a/backend/services/file_management_service.py b/backend/services/file_management_service.py index 585669c0c..b2850403d 100644 --- a/backend/services/file_management_service.py +++ b/backend/services/file_management_service.py @@ -52,27 +52,6 @@ logger = logging.getLogger("file_management_service") -ALLOWED_SKILL_UPLOAD_ROOT = Path("/mnt/nexent").resolve() - - -def is_allowed_skill_upload_path(file_path: str) -> bool: - """Return True when a local file path is under the allowed skill upload root.""" - if not file_path: - return False - - try: - candidate_path = Path(file_path).resolve() - except Exception: - return False - - try: - candidate_path.relative_to(ALLOWED_SKILL_UPLOAD_ROOT) - return True - except ValueError: - return False - - - def resolve_minio_upload_folder( folder: Optional[str], @@ -104,11 +83,6 @@ def resolve_minio_upload_folder( if folder == "knowledge_base": return "knowledge_base" - if folder == "skill-files": - if user_id: - return f"skill-files/{user_id}" - return "skill-files" - if user_id: return f"attachments/{user_id}" @@ -127,6 +101,7 @@ def check_file_access( - knowledge_base/*: All authenticated users can access - attachments/{user_id}/*: Only the owner (user_id) can access - images_in_attachments/*: All authenticated users can access + - preview/*: Accessible if the original file is accessible Args: object_name: File object name in storage @@ -150,10 +125,6 @@ def check_file_access( # Keep them readable for authenticated users to avoid broken image citations. return True - if object_name.startswith("skill-files/"): - # Generated documents are private to the uploader and must stay user-scoped. - return object_name.startswith(f"skill-files/{user_id}/") - # Check if file is in user's attachments folder # Pattern: attachments/{user_id}/* if object_name.startswith(f"attachments/{user_id}/"): @@ -386,20 +357,14 @@ async def upload_to_minio( # Convert file content to BytesIO object file_obj = BytesIO(file_content) - # Store original filename before upload - original_filename = f.filename or "" - # Upload file result = upload_fileobj( file_obj=file_obj, - file_name=original_filename, + file_name=f.filename or "", prefix=actual_folder, file_size=len(file_content) ) - # Preserve original filename in result (upload_fileobj uses it for object name generation) - result["original_file_name"] = original_filename - # Reset file pointer for potential re-reading await f.seek(0) results.append(result) @@ -411,7 +376,6 @@ async def upload_to_minio( results.append({ "success": False, "file_name": f.filename, - "original_file_name": f.filename, "error": "An error occurred while processing the file." }) return results diff --git a/backend/services/northbound_service.py b/backend/services/northbound_service.py index c5493a551..a6eaed77d 100644 --- a/backend/services/northbound_service.py +++ b/backend/services/northbound_service.py @@ -1,40 +1,31 @@ import asyncio import hashlib -import json import logging import time from dataclasses import dataclass -from os.path import basename -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional -from fastapi import HTTPException, UploadFile from fastapi.responses import StreamingResponse - -from consts.const import ASSET_OWNER_TENANT_ID from consts.exceptions import ( LimitExceededError, UnauthorizedError, - ConversationNotFoundError, ) -from consts.model import AgentRequest, ToolParamsRequest -from database.conversation_db import get_conversation_messages, get_source_searches_by_message +from consts.model import AgentRequest +from database.conversation_db import get_conversation_messages from database.token_db import log_token_usage, get_latest_usage_metadata from services.agent_service import ( run_agent_stream, stop_agent_tasks, + list_all_agent_info_impl, get_agent_id_by_name ) -from services.agent_version_service import list_published_agents_impl from services.conversation_management_service import ( save_conversation_user, get_conversation_list_service, create_new_conversation, update_conversation_title as update_conversation_title_service, ) -from services.file_management_service import upload_to_minio, resolve_minio_upload_folder, validate_urls_access -from database.attachment_db import get_file_url, get_file_size_from_minio -from nexent.multi_modal.utils import parse_s3_url logger = logging.getLogger("northbound_service") @@ -48,188 +39,6 @@ class NorthboundContext: token_id: int = 0 -def _build_northbound_file_descriptor( - upload_result: Dict[str, Any], - original_file_name: str = "", - file_type: Optional[str] = None, - file_size: Optional[int] = None, -) -> Dict[str, Any]: - """Normalize upload metadata for northbound API consumers.""" - object_name = str(upload_result.get("object_name") or "").strip() - # Use original filename if provided, otherwise fall back to upload result or object name - if original_file_name: - file_name = original_file_name - else: - file_name = str(upload_result.get("file_name") or basename(object_name) or "") - # Frontend-compatible field order - descriptor = { - "object_name": object_name, - "name": file_name, - "type": file_type or "file", - # Use provided file_size, or from upload_result, or 0 as fallback - "size": file_size if file_size is not None else upload_result.get("file_size", 0), - # Use relative URL format matching frontend: /nexent/{object_name} - "url": f"/nexent/{object_name}", - "description": "", - } - presigned_url = upload_result.get("presigned_url") - if presigned_url: - descriptor["presigned_url"] = presigned_url - return descriptor - - -async def upload_files_for_northbound( - ctx: NorthboundContext, - files: List[UploadFile], - folder: str = "attachments", -) -> Dict[str, Any]: - """Upload files for northbound callers and return reusable storage references.""" - if not files: - raise ValueError("No files in the request") - - actual_folder = resolve_minio_upload_folder(folder, ctx.user_id, ctx.tenant_id) - results = await upload_to_minio(files=files, folder=actual_folder) - normalized_files = [] - for result, upload_file in zip(results, files): - if result.get("success") and result.get("object_name"): - content_type = result.get("content_type", "") - file_type = "image" if content_type.startswith("image/") else "file" - # Extract original filename - use upload result first, then fallback to UploadFile - # The upload result contains the original filename passed to upload_fileobj - original_file_name = result.get("original_file_name") or upload_file.filename or "" - file_size = result.get("file_size", 0) - # If file_size is 0 but we have the UploadFile, try to get size from headers - if file_size == 0 and hasattr(upload_file, 'size') and upload_file.size: - file_size = upload_file.size - descriptor = _build_northbound_file_descriptor( - result, - original_file_name=original_file_name, - file_type=file_type, - file_size=file_size, - ) - normalized_files.append(descriptor) - - if not normalized_files: - raise ValueError("No valid files uploaded") - - success_count = sum(1 for result in results if result.get("success", False)) - failed_count = sum(1 for result in results if not result.get("success", False)) - - return { - "message": f"Processed {len(results)} files", - "requestId": ctx.request_id, - "summary": { - "total": len(results), - "uploaded": success_count, - "failed": failed_count, - }, - "files": normalized_files, - } - - -def _normalize_northbound_attachments( - attachments: Optional[List[Any]], - user_id: str, - tenant_id: str, -) -> Optional[List[Dict[str, Any]]]: - """Convert northbound attachment references into internal minio_files objects. - - Supports two formats: - 1. List of S3 URL strings (backward compatible): ["s3://nexent/...", "/nexent/...", "attachments/..."] - 2. List of attachment objects (full metadata): [{"object_name": "...", "name": "...", ...}] - """ - from database.attachment_db import _build_mcp_presigned_url - - if attachments is None: - return None - if not isinstance(attachments, list): - raise ValueError("attachments must be an array") - - normalized_files: List[Dict[str, Any]] = [] - for attachment in attachments: - # Handle dict format (full attachment object) - if isinstance(attachment, dict): - # Use the attachment dict directly, just ensure required fields - normalized_file = { - "object_name": attachment.get("object_name", ""), - "name": attachment.get("name", basename(attachment.get("object_name", ""))), - "type": attachment.get("type", "file"), - "size": attachment.get("size", 0), - "url": attachment.get("url", ""), - "description": attachment.get("description", ""), - } - # Add presigned_url if available, or generate one if we have object_name - if "presigned_url" in attachment: - normalized_file["presigned_url"] = attachment["presigned_url"] - elif normalized_file.get("object_name"): - try: - presigned_result = get_file_url(object_name=normalized_file["object_name"], expires=86400) - if presigned_result.get("success") and presigned_result.get("url"): - normalized_file["presigned_url"] = _build_mcp_presigned_url(presigned_result["url"]) - except Exception: - pass - normalized_files.append(normalized_file) - continue - - # Handle string format (S3 URL) - if not isinstance(attachment, str) or not attachment.strip(): - raise ValueError("attachments must contain non-empty S3 URLs or object paths") - - attachment_url = attachment.strip() - - # Support multiple URL formats: - # 1. s3://nexent/attachments/xxx.md - # 2. /nexent/attachments/xxx.md - # 3. attachments/xxx.md (relative path) - if attachment_url.startswith("s3://"): - try: - _, object_name = parse_s3_url(attachment_url) - except ValueError as exc: - raise ValueError(f"Invalid S3 URL format: {attachment_url}") from exc - validate_url = attachment_url - elif attachment_url.startswith("/nexent/"): - object_name = attachment_url[len("/nexent/"):] - validate_url = f"s3://nexent/{object_name}" - elif attachment_url.startswith("attachments/") or attachment_url.startswith("nexent/"): - object_name = attachment_url if attachment_url.startswith("nexent/") else attachment_url - validate_url = f"s3://nexent/{object_name}" - else: - raise ValueError(f"Invalid attachment format: {attachment_url}. Expected s3:// URL, /nexent/ path, or attachments/ path") - - try: - validate_urls_access([validate_url], user_id, tenant_id) - presigned_result = get_file_url(object_name=object_name, expires=86400) - except PermissionError as exc: - detail = str(exc) - if "Invalid S3 URL format" in detail: - raise ValueError(detail) from exc - raise PermissionError(detail) from exc - - # Get file size from MinIO - try: - file_size = get_file_size_from_minio(object_name) - except Exception: - file_size = 0 - - # Build frontend-compatible minio_files format - file_name = basename(object_name.rstrip("/")) - normalized_file = { - "object_name": object_name, - "name": file_name, - "type": "file", - "size": file_size, - # Use relative URL format matching frontend: /nexent/{object_name} - "url": f"/nexent/{object_name}", - "description": "", - } - # Use MCP proxy URL for presigned_url (same as frontend format) - if presigned_result.get("success") and presigned_result.get("url"): - normalized_file["presigned_url"] = _build_mcp_presigned_url(presigned_result["url"]) - normalized_files.append(normalized_file) - - return normalized_files - - # ----------------------------- # In-memory idempotency and rate limit placeholders # ----------------------------- @@ -302,12 +111,6 @@ def _build_idempotency_key(*parts: Any) -> str: return ":".join(processed) -def _build_title_update_idempotency_key(tenant_id: str, conversation_id: int, title: str) -> str: - """Build an ASCII-safe idempotency key for title updates.""" - title_hash = hashlib.sha256(title.encode("utf-8")).hexdigest() - return _build_idempotency_key(tenant_id, str(conversation_id), title_hash) - - # ----------------------------- # Agent resolver # ----------------------------- @@ -323,9 +126,7 @@ async def start_streaming_chat( conversation_id: Optional[int], agent_name: str, query: str, - attachments: Optional[List[Any]] = None, meta_data: Optional[Dict[str, Any]] = None, - tool_params: Optional[ToolParamsRequest] = None, idempotency_key: Optional[str] = None ) -> StreamingResponse: try: @@ -344,11 +145,6 @@ async def start_streaming_chat( # Get history according to internal_conversation_id history_resp = await get_conversation_history_internal(ctx, internal_conversation_id) agent_id = await get_agent_id_by_name(agent_name=agent_name, tenant_id=ctx.tenant_id) - normalized_attachments = _normalize_northbound_attachments( - attachments=attachments, - user_id=ctx.user_id, - tenant_id=ctx.tenant_id, - ) # Idempotency: only prevent concurrent duplicate starts composed_key = idempotency_key or _build_idempotency_key(ctx.tenant_id, str(conversation_id), agent_id, query) await idempotency_start(composed_key) @@ -357,9 +153,8 @@ async def start_streaming_chat( agent_id=agent_id, query=query, history=(history_resp.get("data", {})).get("history", []), - minio_files=normalized_attachments, + minio_files=None, is_debug=False, - tool_params=tool_params, ) # Synchronously persist the user message before starting the stream to avoid race conditions @@ -462,58 +257,15 @@ async def list_conversations(ctx: NorthboundContext) -> Dict[str, Any]: return {"message": "success", "data": conversations, "requestId": ctx.request_id} -def _format_search_record(record: Dict[str, Any]) -> Dict[str, Any]: - """Format a search source record for API response.""" - search_item = { - "title": record.get("source_title", ""), - "text": record.get("source_content", ""), - "source_type": record.get("source_type", ""), - "url": record.get("source_location", ""), - "filename": record.get("source_title", "") if record.get("source_type") == "file" else None, - "published_date": None, - "score": float(record["score_overall"]) if record.get("score_overall") is not None else None, - "tool_sign": record.get("tool_sign", ""), - "cite_index": record.get("cite_index") - } - - if record.get("published_date"): - if hasattr(record["published_date"], "strftime"): - search_item["published_date"] = record["published_date"].strftime("%Y-%m-%d") - else: - search_item["published_date"] = str(record["published_date"])[:10] - - return search_item - - async def get_conversation_history_internal(ctx: NorthboundContext, conversation_id: int) -> Dict[str, Any]: """Internal helper to get conversation history without logging.""" history = get_conversation_messages(conversation_id) + # Remove unnecessary fields result = [] for message in history: - # Parse minio_files from database (stored as JSON string) - minio_files = [] - raw_minio_files = message.get("minio_files") - if raw_minio_files: - try: - minio_files = json.loads(raw_minio_files) if isinstance(raw_minio_files, str) else raw_minio_files - except (json.JSONDecodeError, TypeError): - logger.warning(f"Failed to parse minio_files for message {message.get('message_id')}") - - # Fetch search results for this message - message_id = message.get("message_id") - search_results = [] - if message_id: - try: - search_records = get_source_searches_by_message(message_id, user_id=ctx.user_id) - search_results = [_format_search_record(r) for r in search_records] - except Exception as e: - logger.warning(f"Failed to get search records for message {message_id}: {str(e)}") - result.append({ "role": message["message_role"], - "content": message["message_content"], - "minio_files": minio_files, - "search": search_results + "content": message["message_content"] }) response = { @@ -532,18 +284,7 @@ async def get_conversation_history(ctx: NorthboundContext, conversation_id: int) async def get_agent_info_list(ctx: NorthboundContext) -> Dict[str, Any]: try: - agent_info_list = await list_published_agents_impl( - tenant_id=ctx.tenant_id, - user_id=ctx.user_id, - ) - # Match the same scope as /agent/published_list: non-asset-owner tenants - # also get the asset owner's published agents merged in. - if ctx.tenant_id != ASSET_OWNER_TENANT_ID: - asset_agent_list = await list_published_agents_impl( - tenant_id=ASSET_OWNER_TENANT_ID, - user_id=ctx.user_id, - ) - agent_info_list.extend(asset_agent_list) + agent_info_list = await list_all_agent_info_impl(tenant_id=ctx.tenant_id, user_id=ctx.user_id) # Remove internal information that partner don't need for agent_info in agent_info_list: agent_info.pop("agent_id", None) @@ -557,11 +298,7 @@ async def update_conversation_title(ctx: NorthboundContext, conversation_id: int composed_key: Optional[str] = None try: # Idempotency: avoid concurrent duplicate title update for same conversation - composed_key = idempotency_key or _build_title_update_idempotency_key( - ctx.tenant_id, - conversation_id, - title, - ) + composed_key = idempotency_key or _build_idempotency_key(ctx.tenant_id, str(conversation_id), title) await idempotency_start(composed_key) update_conversation_title_service(conversation_id, title, ctx.user_id) @@ -587,8 +324,6 @@ async def update_conversation_title(ctx: NorthboundContext, conversation_id: int } except LimitExceededError as _: raise LimitExceededError("Duplicate request is still running, please wait.") - except ConversationNotFoundError: - raise except Exception as e: raise Exception(f"Failed to update conversation title for conversation_id {conversation_id}: {str(e)}") finally: diff --git a/backend/services/prompt_service.py b/backend/services/prompt_service.py index f1564cdbc..ee9704302 100644 --- a/backend/services/prompt_service.py +++ b/backend/services/prompt_service.py @@ -1,17 +1,15 @@ import json import logging import queue -import sys import threading from typing import Optional, List from jinja2 import StrictUndefined, Template -from consts.const import LANGUAGE, ENABLE_JIUWEN_SDK +from consts.const import LANGUAGE from consts.error_code import ErrorCode from consts.error_message import ErrorMessage from consts.exceptions import AppException -from consts.model import AgentInfoRequest from database.agent_db import search_agent_info_by_agent_id, query_all_agent_info_by_tenant_id, \ query_sub_agents_id_list from database.model_management_db import get_model_by_model_id @@ -24,31 +22,15 @@ _regenerate_agent_name_with_llm, _regenerate_agent_display_name_with_llm, _generate_unique_agent_name_with_suffix, - _generate_unique_display_name_with_suffix, - update_agent, + _generate_unique_display_name_with_suffix ) from services.prompt_template_service import resolve_prompt_generate_template from utils.llm_utils import call_llm_for_system_prompt from utils.prompt_template_utils import ( + get_prompt_generate_prompt_template, get_prompt_optimize_prompt_template, - get_prompt_template, ) -from dataclasses import dataclass, field -from typing import Optional as Opt - -from adapters.exception import JiuwenSDKError, NexentCapabilityError - - -def _get_jiuwen_adapter_class(): - """Import Jiuwen adapter only when optimization paths need it.""" - try: - from adapters import JiuwenSDKAdapter - except ModuleNotFoundError: - return None - return JiuwenSDKAdapter - - # Configure logging logger = logging.getLogger("prompt_service") @@ -123,16 +105,14 @@ def generate_and_save_system_prompt_impl(agent_id: int, # Get knowledge base display names for few-shot examples # Priority: frontend-provided > database query if knowledge_base_display_names: - logger.debug( - f"Using frontend-provided knowledge base display names: {knowledge_base_display_names}") + logger.debug(f"Using frontend-provided knowledge base display names: {knowledge_base_display_names}") else: knowledge_base_display_names = get_knowledge_base_display_names( tool_info_list=tool_info_list, agent_id=agent_id, tenant_id=tenant_id ) - logger.debug( - f"Using database query for knowledge base display names: {knowledge_base_display_names}") + logger.debug(f"Using database query for knowledge base display names: {knowledge_base_display_names}") # Handle sub-agent IDs if sub_agent_ids and len(sub_agent_ids) > 0: @@ -166,7 +146,7 @@ def generate_and_save_system_prompt_impl(agent_id: int, # 1. Real-time streaming push final_results = {"duty": "", "constraint": "", "few_shots": "", "agent_var_name": "", "agent_display_name": "", - "agent_description": "", "greeting_message": "", "example_questions": ""} + "agent_description": ""} # Get all existing agent names and display names for duplicate checking (only if not in create mode) all_agents = query_all_agent_info_by_tenant_id(tenant_id) @@ -212,8 +192,7 @@ def generate_and_save_system_prompt_impl(agent_id: int, exclude_agent_id=agent_id, agents_cache=all_agents ): - logger.info( - f"Agent name '{agent_name}' already exists, regenerating with LLM") + logger.info(f"Agent name '{agent_name}' already exists, regenerating with LLM") try: agent_name = _regenerate_agent_name_with_llm( original_name=agent_name, @@ -227,12 +206,10 @@ def generate_and_save_system_prompt_impl(agent_id: int, prompt_template_id=prompt_template_id, user_id=user_id, ) - logger.info( - f"Regenerated agent name: '{agent_name}'") + logger.info(f"Regenerated agent name: '{agent_name}'") final_results["agent_var_name"] = agent_name except Exception as e: - logger.error( - f"Failed to regenerate agent name with LLM: {str(e)}, using fallback") + logger.error(f"Failed to regenerate agent name with LLM: {str(e)}, using fallback") # Fallback: add suffix agent_name = _generate_unique_agent_name_with_suffix( agent_name, @@ -258,8 +235,7 @@ def generate_and_save_system_prompt_impl(agent_id: int, exclude_agent_id=agent_id, agents_cache=all_agents ): - logger.info( - f"Agent display_name '{agent_display_name}' already exists, regenerating with LLM") + logger.info(f"Agent display_name '{agent_display_name}' already exists, regenerating with LLM") try: agent_display_name = _regenerate_agent_display_name_with_llm( original_display_name=agent_display_name, @@ -273,12 +249,10 @@ def generate_and_save_system_prompt_impl(agent_id: int, prompt_template_id=prompt_template_id, user_id=user_id, ) - logger.info( - f"Regenerated agent display_name: '{agent_display_name}'") + logger.info(f"Regenerated agent display_name: '{agent_display_name}'") final_results["agent_display_name"] = agent_display_name except Exception as e: - logger.error( - f"Failed to regenerate agent display_name with LLM: {str(e)}, using fallback") + logger.error(f"Failed to regenerate agent display_name with LLM: {str(e)}, using fallback") # Fallback: add suffix agent_display_name = _generate_unique_display_name_with_suffix( agent_display_name, @@ -311,68 +285,6 @@ def generate_and_save_system_prompt_impl(agent_id: int, if not has_content: raise Exception("Failed to generate prompt content.") - # 3. Generate greeting message and example questions - try: - greeting_template = get_prompt_template('greeting_generate', language) - greeting_system_prompt = greeting_template.get("GREETING_SYSTEM_PROMPT", "") - greeting_user_prompt_template = greeting_template.get("USER_PROMPT", "") - - greeting_user_prompt = Template(greeting_user_prompt_template, undefined=StrictUndefined).render({ - "display_name": final_results.get("agent_display_name", ""), - "duty_description": final_results.get("duty", ""), - "business_description": task_description, - "few_shots": final_results.get("few_shots", ""), - }) - - greeting_result = call_llm_for_system_prompt( - model_id=model_id, - user_prompt=greeting_user_prompt, - system_prompt=greeting_system_prompt, - tenant_id=tenant_id, - ) - - parsed = None - try: - json_start = greeting_result.find("{") - json_end = greeting_result.rfind("}") + 1 - if json_start >= 0 and json_end > json_start: - parsed = json.loads(greeting_result[json_start:json_end]) - except json.JSONDecodeError: - logger.warning(f"Failed to parse greeting JSON from LLM output: {greeting_result}") - - if parsed and "greeting_message" in parsed and "example_questions" in parsed: - greeting_message = parsed["greeting_message"] - example_questions = parsed["example_questions"] - if isinstance(example_questions, list) and len(example_questions) > 6: - example_questions = example_questions[:6] - else: - greeting_message = greeting_result.strip() if greeting_result else "" - example_questions = [] - - yield { - "type": "greeting_message", - "content": greeting_message, - "is_complete": True - } - yield { - "type": "example_questions", - "content": json.dumps(example_questions, ensure_ascii=False), - "is_complete": True - } - - final_results["greeting_message"] = greeting_message - final_results["example_questions"] = json.dumps(example_questions, ensure_ascii=False) - - # Update agent with greeting (skip in create mode) - if agent_id != 0: - update_agent(agent_id, AgentInfoRequest( - agent_id=agent_id, - greeting_message=greeting_message, - example_questions=example_questions, - ), user_id) - except Exception as e: - logger.warning(f"Greeting generation failed: {str(e)}, skipping greeting") - def optimize_prompt_section_impl( agent_id: int, model_id: int, @@ -427,8 +339,7 @@ def optimize_prompt_section_impl( prompt_context = join_info_for_optimize_prompt_section( prompt_for_optimize=prompt_template, section_type=normalized_section_type, - section_title=section_title or _default_prompt_section_title( - normalized_section_type, language), + section_title=section_title or _default_prompt_section_title(normalized_section_type, language), task_description=task_description, current_content=current_content, feedback=feedback, @@ -487,8 +398,7 @@ def generate_system_prompt(sub_agent_info_list, task_description, tool_info_list # If None or >= 6, no limit (all 6 calls run concurrently) # If < 6, use semaphore to limit concurrent calls model_config = get_model_by_model_id(model_id, tenant_id) - concurrency_limit = model_config.get( - "concurrency_limit") if model_config else None + concurrency_limit = model_config.get("concurrency_limit") if model_config else None # Start all generation threads with concurrency control threads, error_holder = _start_generation_threads( @@ -533,8 +443,7 @@ def _resolve_knowledge_base_display_names( agent_id=agent_id, tenant_id=tenant_id ) - logger.debug( - f"Using database query for knowledge base display names: {resolved_names}") + logger.debug(f"Using database query for knowledge base display names: {resolved_names}") return resolved_names @@ -562,9 +471,8 @@ def _resolve_prompt_generation_sub_agents( tenant_id=tenant_id, agent_id=agent_id ) - def _start_generation_threads(content, prompt_for_generate, produce_queue, latest, stop_flags, tenant_id, model_id, - has_selected_resources=True, concurrency_limit: Optional[int] = None): + has_selected_resources = True, concurrency_limit: Optional[int] = None): """Start all prompt generation threads with optional concurrency control.""" # Shared error tracking across threads error_holder = {"error": None} @@ -580,11 +488,9 @@ def _start_generation_threads(content, prompt_for_generate, produce_queue, lates effective_limit = concurrency_limit # Use semaphore if concurrency is limited - semaphore = threading.Semaphore( - effective_limit) if effective_limit else None + semaphore = threading.Semaphore(effective_limit) if effective_limit else None if semaphore: - logger.info( - f"Using concurrency limit of {effective_limit} for prompt generation (total tasks: {total_tasks})") + logger.info(f"Using concurrency limit of {effective_limit} for prompt generation (total tasks: {total_tasks})") else: logger.info("Using unlimited concurrency for prompt generation") @@ -633,8 +539,7 @@ def run_and_flag(tag, sys_prompt): ("few_shots", prompt_for_generate["few_shots_system_prompt"]), ]) else: - logger.info( - "Skipping constraint and few_shots generation: no tools or sub-agents selected") + logger.info("Skipping constraint and few_shots generation: no tools or sub-agents selected") # Mark these sections as already complete with empty content stop_flags["constraint"] = True stop_flags["few_shots"] = True @@ -733,15 +638,13 @@ def join_info_for_generate_system_prompt(prompt_for_generate, sub_agent_info_lis # This is necessary because Jinja2 StrictUndefined raises an error for any # undefined variable, even inside an {% if %} block. if knowledge_base_display_names: - kb_names_str = ", ".join( - f'"{name}"' for name in knowledge_base_display_names) + kb_names_str = ", ".join(f'"{name}"' for name in knowledge_base_display_names) else: kb_names_str = "" template_context["knowledge_base_names"] = kb_names_str # Generate content using template - content = Template( - prompt_for_generate["user_prompt"], undefined=StrictUndefined).render(template_context) + content = Template(prompt_for_generate["user_prompt"], undefined=StrictUndefined).render(template_context) return content @@ -769,8 +672,7 @@ def join_info_for_optimize_prompt_section( ) if knowledge_base_display_names: - kb_names_str = ", ".join( - f'"{name}"' for name in knowledge_base_display_names) + kb_names_str = ", ".join(f'"{name}"' for name in knowledge_base_display_names) else: kb_names_str = "" @@ -822,8 +724,7 @@ def get_knowledge_base_display_names(tool_info_list: List[dict], agent_id: int, List of knowledge base display names if knowledge_base_search tool is configured, None otherwise """ # Check if knowledge_base_search tool is in the list - kb_tool_ids = [tool['tool_id'] for tool in tool_info_list if tool.get( - 'name') == 'knowledge_base_search'] + kb_tool_ids = [tool['tool_id'] for tool in tool_info_list if tool.get('name') == 'knowledge_base_search'] if not kb_tool_ids: logger.debug("No knowledge_base_search tool found in tool list") return None @@ -846,23 +747,19 @@ def get_knowledge_base_display_names(tool_info_list: List[dict], agent_id: int, try: all_index_names.extend(json.loads(index_names)) except json.JSONDecodeError: - logger.warning( - f"Failed to parse index_names JSON: {index_names}") + logger.warning(f"Failed to parse index_names JSON: {index_names}") except Exception as e: - logger.warning( - f"Failed to get tool instance for tool_id {kb_tool_id}: {e}") + logger.warning(f"Failed to get tool instance for tool_id {kb_tool_id}: {e}") if not all_index_names: - logger.debug( - "No index_names configured for knowledge_base_search tool") + logger.debug("No index_names configured for knowledge_base_search tool") return None # Remove duplicates while preserving order unique_index_names = list(dict.fromkeys(all_index_names)) # Convert to display names - knowledge_name_map = get_knowledge_name_map_by_index_names( - unique_index_names) + knowledge_name_map = get_knowledge_name_map_by_index_names(unique_index_names) # Return list of display names (knowledge_name) for each configured index_name display_names = [] @@ -871,8 +768,7 @@ def get_knowledge_base_display_names(tool_info_list: List[dict], agent_id: int, if display_name and display_name not in display_names: display_names.append(display_name) - logger.debug( - f"Converted index_names {unique_index_names} to display_names: {display_names}") + logger.debug(f"Converted index_names {unique_index_names} to display_names: {display_names}") return display_names if display_names else None @@ -889,299 +785,3 @@ def get_enabled_sub_agent_description_for_generate_prompt(agent_id: int, tenant_ sub_agent_info_list.append(sub_agent_info) return sub_agent_info_list - - -# ── Jiuwen SDK 集成 ─────────────────────────────────────────────────────────── - - -@dataclass -class OptimizeRequest: - """优化请求的统一数据结构""" - agent_id: int - model_id: int - task_description: str - section_type: str - section_title: str - current_content: str - feedback: str - mode: str = "general" - start_pos: Opt[int] = None - end_pos: Opt[int] = None - tool_ids: Opt[list[int]] = None - sub_agent_ids: Opt[list[int]] = None - knowledge_base_display_names: Opt[list[str]] = None - - -@dataclass -class OptimizeResult: - """优化结果的统一数据结构""" - optimized_content: str - source: str - section_type: str = "" - section_title: str = "" - original_content: str = "" - - -class PromptOptimizationService: - """提示词优化服务 — 统一入口，模式二选一""" - - def optimize_from_debug(self, agent_id: int, feedback: str, selected, history=None) -> OptimizeResult: - """基于调试对话自动优化整个 system prompt（完整模板）。 - - Args: - selected: OptimizeFromDebugSelected (pydantic model) or any object with user_question/assistant_answer. - history: Optional[List[HistoryItem]] - """ - if not (feedback or "").strip(): - raise AppException( - ErrorCode.COMMON_MISSING_REQUIRED_FIELD, - "Optimization feedback is required.", - ) - - if not self.is_jiuwen_mode_available(): - raise NexentCapabilityError( - "Auto optimize from debug requires Jiuwen SDK to be enabled." - ) - - agent_info = search_agent_info_by_agent_id( - agent_id=agent_id, tenant_id=self.tenant_id, version_no=0) - - duty = (agent_info.get("duty_prompt") or "").strip() - constraint = (agent_info.get("constraint_prompt") or "").strip() - few_shots = (agent_info.get("few_shots_prompt") or "").strip() - - original_full_prompt = "\n\n".join( - [ - "# Duty\n" + duty, - "# Constraint\n" + constraint, - "# FewShots\n" + few_shots, - ] - ).strip() - - if not original_full_prompt: - raise AppException( - ErrorCode.COMMON_MISSING_REQUIRED_FIELD, - "Agent system prompt is empty.", - ) - - user_question = getattr(selected, "user_question", None) or ( - selected.get("user_question") if isinstance(selected, dict) else "") - assistant_answer = getattr(selected, "assistant_answer", None) or ( - selected.get("assistant_answer") if isinstance(selected, dict) else "") - - bad_case_obj = type("_BadCase", (), {}) - bc = bad_case_obj() - bc.question = user_question or "" - bc.answer = assistant_answer or "" - bc.label = "" - bc.reason = feedback - - adapter_cls = _get_jiuwen_adapter_class() - if adapter_cls is None: - raise JiuwenSDKError("Jiuwen SDK adapter is unavailable") - - adapter = adapter_cls( - model_id=self.model_id, tenant_id=self.tenant_id) - - optimized_full_prompt = adapter.optimize_badcase( - prompt=original_full_prompt, - bad_cases=[bc], - language=self.language, - ) - - return OptimizeResult( - optimized_content=optimized_full_prompt, - source="jiuwen", - section_type="full_prompt", - section_title="system_prompt", - original_content=original_full_prompt, - ) - - def __init__(self, model_id: int, tenant_id: str, language: str): - self.model_id = model_id - self.tenant_id = tenant_id - self.language = language - - def is_jiuwen_mode_available(self) -> bool: - """判断 Jiuwen SDK 模式是否可用""" - if not ENABLE_JIUWEN_SDK: - return False - - return _get_jiuwen_adapter_class() is not None - - def optimize(self, request: OptimizeRequest) -> OptimizeResult: - """统一优化入口 — 优先 Jiuwen SDK，失败则降级 nexent 原生""" - if self.is_jiuwen_mode_available(): - logger.info( - f"[prompt-optimize] mode={request.mode}, using Jiuwen SDK") - try: - return self._optimize_with_jiuwen(request) - except JiuwenSDKError as e: - logger.warning(f"Jiuwen SDK 模式失败，降级到 nexent 原生: {e}") - return self._optimize_with_nexent(request) - else: - return self._optimize_with_nexent(request) - - def _optimize_with_jiuwen(self, request: OptimizeRequest) -> OptimizeResult: - """Jiuwen SDK 模式""" - logger.info( - f"[jiuwen-optimize] mode={request.mode}, start_pos={request.start_pos}, " - f"end_pos={request.end_pos}, prompt_len={len(request.current_content)}, " - f"feedback_len={len(request.feedback)}" - ) - adapter_cls = _get_jiuwen_adapter_class() - if adapter_cls is None: - raise JiuwenSDKError("Jiuwen SDK adapter is unavailable") - - adapter = adapter_cls( - model_id=self.model_id, - tenant_id=self.tenant_id, - ) - result = adapter.optimize( - prompt=request.current_content, - feedback=request.feedback, - mode=request.mode, - start_pos=request.start_pos, - end_pos=request.end_pos, - language=self.language, - ) - - # Jiuwen insert/select mode returns a fragment by design. - # We reassemble the full prompt here so frontend always receives full optimized content. - if request.mode == "insert": - if request.start_pos is None or not isinstance(request.start_pos, int): - raise JiuwenSDKError("insert mode requires start_pos") - if request.start_pos < 0 or request.start_pos > len(request.current_content): - raise JiuwenSDKError("insert mode start_pos out of bounds") - optimized_full = ( - request.current_content[: request.start_pos] - + result - + request.current_content[request.start_pos:] - ) - elif request.mode == "select": - if request.start_pos is None or request.end_pos is None: - raise JiuwenSDKError( - "select mode requires start_pos and end_pos") - if not isinstance(request.start_pos, int) or not isinstance(request.end_pos, int): - raise JiuwenSDKError( - "select mode start_pos/end_pos must be int") - if request.start_pos < 0 or request.end_pos < 0 or request.start_pos >= request.end_pos: - raise JiuwenSDKError("select mode start_pos/end_pos invalid") - if request.end_pos > len(request.current_content): - raise JiuwenSDKError("select mode end_pos out of bounds") - optimized_full = ( - request.current_content[: request.start_pos] - + result - + request.current_content[request.end_pos:] - ) - else: - optimized_full = result - - return OptimizeResult( - optimized_content=optimized_full, - source="jiuwen", - section_type=request.section_type, - section_title=request.section_title, - original_content=request.current_content, - ) - - def _optimize_with_nexent(self, request: OptimizeRequest) -> OptimizeResult: - """nexent 原生模式 — 只支持 general 模式""" - if request.mode != "general": - raise NexentCapabilityError( - f"nexent 原生模式只支持 general 模式，" - f"当前请求 mode={request.mode} 不支持，请启用 Jiuwen SDK" - ) - - result = optimize_prompt_section_impl( - agent_id=request.agent_id, - model_id=self.model_id, - task_description=request.task_description, - tenant_id=self.tenant_id, - language=self.language, - section_type=request.section_type, - section_title=request.section_title, - current_content=request.current_content, - feedback=request.feedback, - tool_ids=request.tool_ids, - sub_agent_ids=request.sub_agent_ids, - knowledge_base_display_names=request.knowledge_base_display_names, - ) - return OptimizeResult( - optimized_content=result["optimized_content"], - source="nexent", - section_type=result["section_type"], - section_title=result["section_title"], - original_content=result["original_content"], - ) - - def optimize_badcase( - self, - current_content: str, - bad_cases: list, - agent_id: int, - section_type: str, - section_title: str, - tool_ids: Opt[list[int]] = None, - sub_agent_ids: Opt[list[int]] = None, - knowledge_base_display_names: Opt[list[str]] = None, - ) -> OptimizeResult: - """坏案例优化入口 — 优先 Jiuwen SDK，失败则降级""" - if self.is_jiuwen_mode_available(): - logger.info("[prompt-badcase] using Jiuwen SDK") - try: - return self._optimize_badcase_with_jiuwen( - current_content, bad_cases, section_type, section_title - ) - except JiuwenSDKError as e: - logger.warning(f"Jiuwen SDK badcase 模式失败，降级到 nexent 原生: {e}") - return self._optimize_badcase_with_nexent( - current_content, bad_cases, agent_id, section_type, section_title, - tool_ids, sub_agent_ids, knowledge_base_display_names, - ) - else: - return self._optimize_badcase_with_nexent( - current_content, bad_cases, agent_id, section_type, section_title, - tool_ids, sub_agent_ids, knowledge_base_display_names, - ) - - def _optimize_badcase_with_jiuwen( - self, current_content: str, bad_cases: list, section_type: str, section_title: str - ) -> OptimizeResult: - """Jiuwen SDK 坏案例优化""" - adapter_cls = _get_jiuwen_adapter_class() - if adapter_cls is None: - raise JiuwenSDKError("Jiuwen SDK adapter is unavailable") - - adapter = adapter_cls( - model_id=self.model_id, - tenant_id=self.tenant_id, - ) - result = adapter.optimize_badcase( - prompt=current_content, - bad_cases=bad_cases, - language=self.language, - ) - return OptimizeResult( - optimized_content=result, - source="jiuwen", - section_type=section_type, - section_title=section_title, - original_content=current_content, - ) - - def _optimize_badcase_with_nexent( - self, - current_content: str, - bad_cases: list, - agent_id: int, - section_type: str, - section_title: str, - tool_ids: Opt[list[int]] = None, - sub_agent_ids: Opt[list[int]] = None, - knowledge_base_display_names: Opt[list[str]] = None, - ) -> OptimizeResult: - """nexent 原生模式不支持坏案例优化""" - raise NexentCapabilityError( - "nexent 原生模式不支持 badcase 优化，请启用 Jiuwen SDK" - ) diff --git a/backend/services/remote_mcp_service.py b/backend/services/remote_mcp_service.py index 7e77a9c43..56a73fb4b 100644 --- a/backend/services/remote_mcp_service.py +++ b/backend/services/remote_mcp_service.py @@ -230,7 +230,7 @@ async def add_mcp_service( server_url: str, tags: list | None, authorization_token: str | None, - custom_headers: dict | None = None, + custom_headers: dict | None, container_config: dict | None, registry_json: dict | None, enabled: bool = False, diff --git a/backend/services/tool_configuration_service.py b/backend/services/tool_configuration_service.py index 3cbf5edc5..ba51567dc 100644 --- a/backend/services/tool_configuration_service.py +++ b/backend/services/tool_configuration_service.py @@ -782,8 +782,6 @@ def _validate_local_tool( 'embedding_model': embedding_model, 'rerank_model': rerank_model, 'display_name_to_index_map': display_name_to_index_map, - # Internal access control: restrict results to specific document paths (path_or_urls) - 'document_paths': instantiation_params.get('document_paths'), } tool_instance = tool_class(**params) elif tool_name in ["dify_search", "datamate_search"]: @@ -984,7 +982,6 @@ def import_openapi_service( tenant_id: str, user_id: str, service_description: str = None, - headers_template: Dict[str, Any] = None, force_update: bool = False ) -> Dict[str, Any]: """ @@ -998,7 +995,6 @@ def import_openapi_service( tenant_id: Tenant ID for multi-tenancy user_id: User ID for audit service_description: Optional service description (if not provided, reads from openapi_json.info.description) - headers_template: Optional default headers template force_update: If True, replace all existing tools for this service Returns: @@ -1019,8 +1015,7 @@ def import_openapi_service( server_url=server_url, tenant_id=tenant_id, user_id=user_id, - description=service_description, - headers_template=headers_template, + description=service_description ) logger.info(f"Imported service '{service_name}' for tenant {tenant_id}") diff --git a/backend/services/user_management_service.py b/backend/services/user_management_service.py index 0b38a76bc..a983b25d3 100644 --- a/backend/services/user_management_service.py +++ b/backend/services/user_management_service.py @@ -18,7 +18,6 @@ get_supabase_admin_client, calculate_expires_at, get_jwt_expiry_seconds, - ensure_cas_session_active_from_authorization, resolve_tenant_id_from_user_tenant_record, ) from consts.const import ( @@ -108,7 +107,6 @@ def validate_token(token: str) -> Tuple[bool, Optional[Any]]: try: user = get_current_user_from_client(client, token) if user: - ensure_cas_session_active_from_authorization(token) return True, user return False, None except Exception as e: diff --git a/backend/services/vectordatabase_service.py b/backend/services/vectordatabase_service.py index dd2f6e51a..11c5fd9bf 100644 --- a/backend/services/vectordatabase_service.py +++ b/backend/services/vectordatabase_service.py @@ -10,7 +10,6 @@ 4. Health check interface """ import asyncio -import hashlib import json import logging import os @@ -29,7 +28,7 @@ from consts.const import DATAMATE_URL, ES_API_KEY, ES_HOST, LANGUAGE, VectorDatabaseType, IS_SPEED_MODE, PERMISSION_EDIT, PERMISSION_READ, ASSET_OWNER_TENANT_ID from consts.model import ChunkCreateRequest, ChunkUpdateRequest -from database.attachment_db import delete_file, file_exists, get_file_stream +from database.attachment_db import delete_file, get_file_stream from database.knowledge_db import ( create_knowledge_record, delete_knowledge_record, @@ -354,18 +353,15 @@ def get_embedding_model( tenant_id: Tenant ID model_name: Optional display name of the embedding model to use. If provided, will find the model by display_name in the tenant's model list. - model_type: Optional model type filter. When model_name is omitted, queries tenant - model records by this type; when model_type is also omitted, prefers - embedding models, then multi_embedding models. Returns: Tuple of (embedding model instance or None, model_id or None) """ if model_name: try: - model_type = _normalize_model_type(model_type) - if model_type: - model = get_model_by_display_name(model_name, tenant_id, model_type) + normalized_model_type = _normalize_model_type(model_type) + if normalized_model_type: + model = get_model_by_display_name(model_name, tenant_id, normalized_model_type) else: model = get_model_by_display_name(model_name, tenant_id) @@ -376,25 +372,8 @@ def get_embedding_model( return _create_embedding_model(model), model.get("model_id") except Exception as e: logger.warning(f"Failed to get embedding model by name {model_name}: {e}") - else: - try: - if model_type: - records = get_model_records({"model_type": model_type}, tenant_id) - else: - records = get_model_records({"model_type": "embedding"}, tenant_id) - if not records: - records = get_model_records({"model_type": "multi_embedding"}, tenant_id) - - if records: - model = records[0] - if model.get("model_type") in ["embedding", "multi_embedding"]: - return _create_embedding_model(model), model.get("model_id") - logger.warning( - f"Resolved model is not an embedding model: {model.get('model_type')}" - ) - except Exception as e: - logger.warning(f"Failed to get default embedding model for tenant {tenant_id}: {e}") + # No default fallback - return None, None when no model is specified or found return None, None @@ -657,7 +636,6 @@ def create_knowledge_base( group_ids: Optional[List[int]] = None, embedding_model_name: Optional[str] = None, is_multimodal: Optional[bool] = None, - preserve_source_file: Optional[bool] = None, ): """ Create a new knowledge base with a user-facing name and an internal Elasticsearch index name. @@ -677,8 +655,6 @@ def create_knowledge_base( group_ids: List of group IDs (optional) embedding_model_name: Specific embedding model name to use (optional). If provided, will use this model instead of tenant default. - preserve_source_file: Whether to preserve uploaded source documents after - vectorization (optional; defaults to True when omitted). For backward compatibility, legacy callers can still use create_index() directly with an explicit index_name. @@ -718,8 +694,6 @@ def create_knowledge_base( knowledge_data["ingroup_permission"] = ingroup_permission if group_ids is not None: knowledge_data["group_ids"] = group_ids - if preserve_source_file is not None: - knowledge_data["preserve_source_file"] = preserve_source_file record_info = create_knowledge_record(knowledge_data) index_name = record_info["index_name"] @@ -1117,7 +1091,6 @@ def list_indices( # Auto-summary settings "summary_frequency": record.get("summary_frequency"), "last_summary_time": record.get("last_summary_time"), - "preserve_source_file": record.get("preserve_source_file", True), "stats": index_stats, }) @@ -1515,11 +1488,6 @@ async def list_files( # chunk_count is already set from ES aggregation (doc_count) file_data['chunk_count'] = file_data.get('chunk_count', 0) - for file_data in files: - file_data["source_available"] = ( - ElasticSearchService._compute_source_available(file_data) - ) - total_duration = time.time() - total_start_time logger.info(f"[list_files:complete] index={index_name}, total_files={len(files)}, " f"total_duration={total_duration:.3f}s") @@ -1530,100 +1498,6 @@ async def list_files( raise Exception( f"Error getting file list for index {index_name}: {str(e)}") - DOCUMENT_DELETE_SCOPES = ("source_only", "full") - - @staticmethod - def _preview_pdf_cache_object_name(object_name: str) -> str: - """Object key for Office-to-PDF preview cache (matches file_management_service).""" - name_without_ext = ( - object_name.rsplit(".", 1)[0] if "." in object_name else object_name - ) - hash_suffix = hashlib.md5(object_name.encode()).hexdigest()[:8] - return f"preview/converted/{name_without_ext}_{hash_suffix}.pdf" - - @staticmethod - def _compute_source_available(file_data: Dict[str, Any]) -> bool: - path_or_url = file_data.get("path_or_url") or "" - status = file_data.get("status", "") - if status != "COMPLETED": - return True - if path_or_url.startswith("knowledge_base/"): - return file_exists(path_or_url) - return True - - @staticmethod - def delete_source_file(path_or_url: str) -> Dict[str, Any]: - """Remove MinIO source (and preview cache); does not touch Elasticsearch.""" - minio_result = delete_file(path_or_url) - deleted_minio = bool(minio_result.get("success")) - - if path_or_url.startswith("knowledge_base/"): - preview_key = ElasticSearchService._preview_pdf_cache_object_name( - path_or_url - ) - try: - if file_exists(preview_key): - delete_file(preview_key) - except Exception as exc: - logger.warning( - "Failed to delete preview cache for '%s': %s", - path_or_url, - exc, - ) - - return {"deleted_minio": deleted_minio} - - @staticmethod - async def _assert_source_only_deletable( - index_name: str, path_or_url: str - ) -> None: - celery_task_files = await get_all_files_status(index_name) - status_info = celery_task_files.get(path_or_url) - if not status_info or not isinstance(status_info, dict): - return - state = status_info.get("state") or "" - if state and state != "COMPLETED": - raise ValueError( - f"Cannot delete source file while document is in state '{state}'. " - "Wait until processing completes or use scope=full to remove the document." - ) - - @staticmethod - async def delete_document_by_scope( - index_name: str, - path_or_url: str, - scope: str, - vdb_core: VectorDatabaseCore, - ) -> Dict[str, Any]: - if scope not in ElasticSearchService.DOCUMENT_DELETE_SCOPES: - raise ValueError( - f"Invalid scope '{scope}'. " - f"Must be one of: {ElasticSearchService.DOCUMENT_DELETE_SCOPES}" - ) - - if scope == "source_only": - await ElasticSearchService._assert_source_only_deletable( - index_name, path_or_url - ) - minio_part = ElasticSearchService.delete_source_file(path_or_url) - return { - "status": "success", - "scope": scope, - "deleted_es_count": 0, - "deleted_minio": minio_part.get("deleted_minio", False), - "source_available": False, - "message": ( - "Source file deleted; index chunks and vectors preserved." - ), - } - - result = ElasticSearchService.delete_documents( - index_name, path_or_url, vdb_core - ) - result["scope"] = scope - result["source_available"] = False - return result - @staticmethod def delete_documents( index_name: str = Path(..., description="Name of the index"), diff --git a/backend/utils/auth_utils.py b/backend/utils/auth_utils.py index a7194f050..04e81e6e3 100644 --- a/backend/utils/auth_utils.py +++ b/backend/utils/auth_utils.py @@ -326,13 +326,16 @@ def calculate_expires_at(token: Optional[str] = None) -> int: return int((datetime.now() + timedelta(seconds=expiry_seconds)).timestamp()) -def _decode_jwt_token(authorization: str) -> dict: +def _extract_user_id_from_jwt_token(authorization: str) -> Optional[str]: """ Extract user ID from JWT token after verifying signature and expiration. Args: authorization: Authorization header value + Returns: + Optional[str]: User ID, return None if parsing fails + Raises: UnauthorizedError: If token is invalid, expired, or signature verification fails """ @@ -352,12 +355,17 @@ def _decode_jwt_token(authorization: str) -> dict: # Decode and verify JWT (signature + expiration) # verify_aud=False: allow tokens with aud claim (e.g. test JWT, Supabase) without strict audience check - return jwt.decode( + decoded = jwt.decode( token, SUPABASE_JWT_SECRET, algorithms=["HS256"], options={"verify_exp": True, "verify_aud": False}, ) + + # Extract user ID from JWT claims + user_id = decoded.get("sub") + + return user_id except jwt.ExpiredSignatureError: logging.warning("Token expired") raise UnauthorizedError("Token has expired") @@ -370,47 +378,10 @@ def _decode_jwt_token(authorization: str) -> dict: except UnauthorizedError: raise except Exception as e: - logging.error(f"Failed to decode token: {str(e)}") + logging.error(f"Failed to extract user ID from token: {str(e)}") raise UnauthorizedError("Invalid or expired authentication token") -def _extract_user_id_from_jwt_token(authorization: str) -> Optional[str]: - """ - Extract user ID from JWT token after verifying signature and expiration. - """ - decoded = _decode_jwt_token(authorization) - return decoded.get("sub") - - -def extract_session_id_from_authorization(authorization: Optional[str]) -> Optional[str]: - """Extract the sid claim without enforcing token validity, for idempotent logout.""" - if not authorization: - return None - try: - token = ( - authorization.replace("Bearer ", "") - if authorization.startswith("Bearer ") - else authorization - ) - decoded = jwt.decode(token, options={"verify_signature": False}) - sid = decoded.get("sid") - return str(sid) if sid else None - except Exception: - return None - - -def ensure_cas_session_active_from_authorization(authorization: Optional[str]) -> None: - """Reject CAS-issued JWTs whose server-side session is expired or revoked.""" - session_id = extract_session_id_from_authorization(authorization) - if not session_id: - return - - from database.cas_session_db import is_cas_session_active - - if not is_cas_session_active(str(session_id)): - raise UnauthorizedError("CAS session has expired or been revoked") - - def get_current_user_id(authorization: Optional[str] = None) -> tuple[str, str]: """ Get current user ID and tenant ID from authorization token @@ -434,13 +405,10 @@ def get_current_user_id(authorization: Optional[str] = None) -> tuple[str, str]: raise UnauthorizedError("No authorization header provided") try: - decoded = _decode_jwt_token(authorization) - user_id = decoded.get("sub") + user_id = _extract_user_id_from_jwt_token(authorization) if not user_id: raise UnauthorizedError("Invalid or expired authentication token") - ensure_cas_session_active_from_authorization(authorization) - user_tenant_record = get_user_tenant_by_user_id(user_id) if user_tenant_record and user_tenant_record.get("tenant_id"): tenant_id = user_tenant_record["tenant_id"] @@ -453,8 +421,6 @@ def get_current_user_id(authorization: Optional[str] = None) -> tuple[str, str]: return user_id, tenant_id - except UnauthorizedError: - raise except Exception as e: logging.error(f"Failed to get user ID and tenant ID: {str(e)}") raise UnauthorizedError("Invalid or expired authentication token") @@ -506,7 +472,7 @@ def generate_test_jwt(user_id: str, expires_in: int = 3600) -> str: return jwt.encode(payload, MOCK_JWT_SECRET_KEY, algorithm="HS256") -def generate_session_jwt(user_id: str, expires_in: int = 3600, session_id: str = None) -> str: +def generate_session_jwt(user_id: str, expires_in: int = 3600) -> str: """Generate a signed JWT compatible with the existing auth verification flow.""" now = int(time.time()) payload = { @@ -517,8 +483,6 @@ def generate_session_jwt(user_id: str, expires_in: int = 3600, session_id: str = "exp": now + expires_in, "iss": SUPABASE_URL, } - if session_id: - payload["sid"] = session_id return jwt.encode(payload, SUPABASE_JWT_SECRET, algorithm="HS256") diff --git a/backend/utils/context_utils.py b/backend/utils/context_utils.py index 0c3af8915..740bf66df 100644 --- a/backend/utils/context_utils.py +++ b/backend/utils/context_utils.py @@ -8,6 +8,7 @@ allowing ContextManager to assemble them in the correct order. """ +from datetime import datetime from typing import TYPE_CHECKING, Any, Dict, List, Optional if TYPE_CHECKING: @@ -507,12 +508,13 @@ def _format_agent_fallback( return "- 当前没有可用的助手" if language == "zh" else "- No agents are currently available" -def _format_app_context(app_name: str, app_description: str, user_id: str) -> str: +def _format_app_context(app_name: str, app_description: str, user_id: str, time_str: str) -> str: """Format application context for system prompt injection.""" lines = [ f"Application: {app_name}", f"Description: {app_description}", f"Current user: {user_id}", + f"Current time: {time_str}", ] return "\n".join(lines) @@ -526,6 +528,7 @@ def _format_app_context(app_name: str, app_description: str, user_id: str) -> st def build_skeleton_header_component( app_name: str, app_description: str, + time_str: str, user_id: str, language: str = "zh", priority: int = 100, @@ -533,17 +536,14 @@ def build_skeleton_header_component( """Build SystemPromptComponent for the header section. Section: "### 基本信息" / "### Basic Information" - Content: Agent identity, app name/description, user_id. - Note: Current time is intentionally excluded from the system prompt so the - static system prefix can hit the LLM KV/prompt cache across requests. The - current time is injected on the user-message side instead (see CoreAgent.run). + Content: Agent identity, app name/description, time, user_id """ from nexent.core.agents.agent_model import SystemPromptComponent if language == "zh": - content = f"### 基本信息\n你是{app_name}，{app_description}，用户ID为{user_id}" + content = f"### 基本信息\n你是{app_name}，{app_description}，现在是{time_str}，用户ID为{user_id}" else: - content = f"### Basic Information\nYou are {app_name}, {app_description}" + content = f"### Basic Information\nYou are {app_name}, {app_description}, it is {time_str} now" return SystemPromptComponent( content=content, @@ -611,11 +611,6 @@ def build_skeleton_execution_flow_component( lines.append(" - 注意运行的代码不会被用户看到，所以如果用户需要看到代码，你需要使用'代码'表达展示代码。") lines.append(" - **重要**：代码执行后，系统会返回 \"Observation:\" 标记的内容（这是真实的执行结果）。请基于这些真实结果继续下一步思考，**不要在代码执行前自行编造观察结果**。") lines.append("") - lines.append("3. 自验证：") - lines.append(" - 关键事件（工具调用、检索结果、代码执行、助手返回、准备最终回答）后，系统会进行显式自验证。") - lines.append(" - 如果自验证提示存在错误、证据不足、参数不完整或结果不可靠，必须优先修正、补充证据、重新调用工具，或清晰说明无法完成的部分。") - lines.append(" - 最终回答只有在自验证通过后才会展示给用户；如果系统返回 Verification feedback，请把它视为真实观察结果继续修正，不要忽略。") - lines.append("") lines.append("在思考结束后，当你认为可以回答用户问题，那么可以不生成代码，直接生成最终回答给到用户并停止循环。") lines.append("") lines.append("生成最终回答时，你需要遵循以下规范：") @@ -657,11 +652,6 @@ def build_skeleton_execution_flow_component( lines.append(" - Note that executed code is not visible to users. If users need to see the code, use 'code' for displaying code.") lines.append(" - **IMPORTANT**: After code execution, the system will return content with \"Observation:\" marker (this is the real execution result). Please continue your next thinking based on these real results. **Do NOT fabricate observation results before code execution.**") lines.append("") - lines.append("3. Self-verification:") - lines.append(" - After critical events (tool calls, retrieval results, code execution, agent handoffs, and final-answer preparation), the system may run explicit verification.") - lines.append(" - If verification reports errors, insufficient evidence, incomplete parameters, or unreliable results, you must repair the issue, gather more evidence, call tools again, or clearly state what cannot be completed.") - lines.append(" - The final answer is shown to the user only after verification passes. If the system returns Verification feedback, treat it as a real observation and continue revising.") - lines.append("") lines.append("After thinking, when you believe you can answer the user's question, you can generate a final answer directly to the user without generating code and stop the loop.") lines.append("") lines.append("When generating the final answer, you need to follow these specifications:") @@ -1122,6 +1112,7 @@ def build_context_components( few_shots: Optional[str] = None, app_name: Optional[str] = None, app_description: Optional[str] = None, + time_str: Optional[str] = None, user_id: Optional[str] = None, language: str = "zh", is_manager: bool = True, @@ -1176,6 +1167,7 @@ def build_context_components( few_shots: Example templates text app_name: Application name app_description: Application description + time_str: Current time string user_id: Current user ID language: Language code ('zh' or 'en') is_manager: Whether this is a manager agent @@ -1196,11 +1188,12 @@ def build_context_components( components: List = [] # 1. Header - if app_name and app_description and user_id: + if app_name and app_description and time_str and user_id: components.append( build_skeleton_header_component( app_name=app_name, app_description=app_description, + time_str=time_str, user_id=user_id, language=language, ) @@ -1335,4 +1328,5 @@ def build_app_context_string( Returns: Formatted app context string """ - return _format_app_context(app_name, app_description, user_id) + time_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + return _format_app_context(app_name, app_description, user_id, time_str) \ No newline at end of file diff --git a/backend/utils/http_client_utils.py b/backend/utils/http_client_utils.py index 262c0a593..1c1d14af6 100644 --- a/backend/utils/http_client_utils.py +++ b/backend/utils/http_client_utils.py @@ -8,7 +8,6 @@ def create_httpx_client( headers: dict[str, str] | None = None, timeout: httpx.Timeout | None = None, auth: httpx.Auth | None = None, - **kwargs, ) -> AsyncClient: return AsyncClient( headers=headers, @@ -16,5 +15,4 @@ def create_httpx_client( auth=auth, trust_env=False, verify=False, - **kwargs, ) diff --git a/backend/utils/memory_utils.py b/backend/utils/memory_utils.py index e3ba01d6d..ada7019a1 100644 --- a/backend/utils/memory_utils.py +++ b/backend/utils/memory_utils.py @@ -1,5 +1,4 @@ import logging -import re from typing import Dict, Any from urllib.parse import urlparse @@ -10,11 +9,6 @@ logger = logging.getLogger("memory_utils") -def _sanitize_index_component(value: str) -> str: - """Convert arbitrary text into an Elasticsearch-safe index component.""" - return re.sub(r"[^a-z0-9_.-]", "_", value.lower()) - - def build_memory_config(tenant_id: str) -> Dict[str, Any]: """Return a fully-validated configuration dictionary for *mem0* ``Memory``. """ @@ -36,8 +30,9 @@ def build_memory_config(tenant_id: str) -> Dict[str, Any]: es_host = f"{parsed.scheme}://{parsed.hostname}" es_port = parsed.port # Normalize repo/name to avoid problematic characters in index names - safe_repo = _sanitize_index_component(embed_raw["model_repo"]) if embed_raw["model_repo"] else "" - safe_name = _sanitize_index_component(embed_raw["model_name"]) + safe_repo = embed_raw["model_repo"].lower().replace( + "/", "_") if embed_raw["model_repo"] else "" + safe_name = embed_raw["model_name"].lower().replace("/", "_") index_name = ( f"mem0_{safe_repo}_{safe_name}_{embed_raw['max_tokens']}" if embed_raw["model_repo"] @@ -78,4 +73,4 @@ def build_memory_config(tenant_id: str) -> Dict[str, Any]: }, "telemetry": {"enabled": False}, } - return memory_config + return memory_config \ No newline at end of file diff --git a/backend/utils/prompt_template_utils.py b/backend/utils/prompt_template_utils.py index 299d3bf94..8822e5fd4 100644 --- a/backend/utils/prompt_template_utils.py +++ b/backend/utils/prompt_template_utils.py @@ -99,10 +99,6 @@ def get_prompt_template(template_type: str, language: str = LANGUAGE["ZH"], **kw LANGUAGE["ZH"]: 'backend/prompts/utils/generate_title_zh.yaml', LANGUAGE["EN"]: 'backend/prompts/utils/generate_title_en.yaml' }, - 'greeting_generate': { - LANGUAGE["ZH"]: 'backend/prompts/utils/greeting_generate_zh.yaml', - LANGUAGE["EN"]: 'backend/prompts/utils/greeting_generate_en.yaml' - }, 'document_summary': { LANGUAGE["ZH"]: 'backend/prompts/document_summary_agent_zh.yaml', LANGUAGE["EN"]: 'backend/prompts/document_summary_agent_en.yaml' diff --git a/doc/docs/en/quick-start/installation.md b/doc/docs/en/quick-start/installation.md index 7b6a9cb76..0b1544819 100644 --- a/doc/docs/en/quick-start/installation.md +++ b/doc/docs/en/quick-start/installation.md @@ -273,114 +273,6 @@ Provider enablement rules: For local Docker, a GitHub callback example is `http://localhost:3000/api/user/oauth/callback?provider=github`. In production, use a public HTTPS domain such as `https://nexent.example.com/api/user/oauth/callback?provider=github` and register the exact same URL in the OAuth provider console. -### CAS Login Configuration - -CAS SSO does not require the `supabase` component. Set `CAS_CALLBACK_BASE_URL` to the browser-accessible Nexent Web URL without a trailing `/`. `CAS_SERVER_URL` is the CAS Server root URL and should also not include a trailing `/`. - -For Docker, configure CAS in `docker/.env`: - -```bash -CAS_ENABLED=true -CAS_SERVER_URL=http://localhost:8080/cas -CAS_VALIDATE_PATH=/p3/serviceValidate -CAS_CALLBACK_BASE_URL=http://localhost:3000 - -# disabled: disable the CAS login entry and automatic redirects -# button: show CAS as an optional login button -# force: redirect unauthenticated Nexent users to CAS automatically -CAS_LOGIN_MODE=force - -# Empty means use ; set userName to read -CAS_USER_ATTRIBUTE= -CAS_EMAIL_ATTRIBUTE=email -CAS_ROLE_ATTRIBUTE=role -CAS_TENANT_ATTRIBUTE=tenant_id -CAS_ROLE_MAP_JSON={"cas-admin":"ADMIN","cas-user":"USER"} -CAS_SESSION_MAX_AGE_SECONDS=3600 -LOCAL_SESSION_MAX_AGE_SECONDS=3600 -CAS_RENEW_BEFORE_SECONDS=300 -CAS_RENEW_TIMEOUT_SECONDS=10 -CAS_SYNTHETIC_EMAIL_DOMAIN=cas.local - -# Empty means Nexent logout will not call the CAS Server logout endpoint. -# /logout is resolved against CAS_SERVER_URL. -CAS_LOGOUT_URL=/logout -CAS_SSL_VERIFY=true -CAS_CA_BUNDLE= -``` - -Common CAS URLs: - -| Purpose | URL | -|---------|-----| -| Nexent login entry | `{CAS_CALLBACK_BASE_URL}/api/user/cas/login?redirect=/` | -| CAS service callback | `{CAS_CALLBACK_BASE_URL}/api/user/cas/callback` | -| CAS silent renewal callback | `{CAS_CALLBACK_BASE_URL}/api/user/cas/renew_callback` | -| CAS single logout callback | `POST {CAS_CALLBACK_BASE_URL}/api/user/cas/logout_callback` | - -For Apereo CAS JSON Service Registry, create a service registration file such as `Nexent-10001.json` in the service registry directory configured by your CAS deployment. The `id` must be globally unique. This is a local Docker example: - -```json -{ - "@class": "org.apereo.cas.services.RegexRegisteredService", - "serviceId": "http://localhost:3000.*", - "name": "Nexent CAS Client", - "id": 10001, - "description": "Nexent CAS SSO client", - "evaluationOrder": 1, - "logoutType": "BACK_CHANNEL", - "logoutUrl": "http://localhost:3000/api/user/cas/logout_callback" -} -``` - -In production, keep `CAS_SSL_VERIFY=true`; for self-signed certificates, prefer `CAS_CA_BUNDLE` and only use `CAS_SSL_VERIFY=false` for local testing. - -#### CAS Integration with ModelEngine - -When integrating with ModelEngine through the CAS protocol, deploy Nexent with the following configuration: - -```bash -CAS_ENABLED=true -CAS_SERVER_URL=https://:5443/SSOSvr -CAS_VALIDATE_PATH=/p3/serviceValidate -CAS_CALLBACK_BASE_URL=http://:3000 -CAS_LOGIN_MODE=force -CAS_USER_ATTRIBUTE=userName -CAS_EMAIL_ATTRIBUTE=email -CAS_ROLE_ATTRIBUTE=userType -CAS_TENANT_ATTRIBUTE=tenant_id -CAS_ROLE_MAP_JSON={"1":"ADMIN","3":"DEV"} -CAS_SESSION_MAX_AGE_SECONDS=3600 -LOCAL_SESSION_MAX_AGE_SECONDS=3600 -CAS_RENEW_BEFORE_SECONDS=300 -CAS_RENEW_TIMEOUT_SECONDS=10 -CAS_SYNTHETIC_EMAIL_DOMAIN=cas.local -CAS_LOGOUT_URL=/logout?service=http://:3000 -CAS_SSL_VERIFY=false -CAS_CA_BUNDLE= -``` - -You also need to add a CAS client service registration file in the OMS container. Use the following steps as a reference: - -```bash -# Create the registration file, paste the JSON content into it, and save it. -vim Nexent-10000001.json -{ - "@class": "org.apereo.cas.services.CasRegisteredService", - "serviceId": "http://:3000.*", - "name": "Nexent CAS Client", - "id": 1000001, - "description": "Nexent CAS SSO client", - "evaluationOrder": 1, - "logoutType": "BACK_CHANNEL", - "logoutUrl": "http://:3000/api/user/cas/logout_callback" -} - -# Run the following command to copy the registration file into the container. -kubectl cp Nexent-10000001.json model-engine/$(kubectl get pods -n model-engine -l app=oms --no-headers | awk '{print $1}'):/opt/huawei/fce/apps/platform/webapps/SSOSvr/WEB-INF/classes/services/Nexent-10000001.json -kubectl exec -i -n model-engine $(kubectl get pods -n model-engine -l app=oms --no-headers | awk '{print $1}') -- chown tomcat:fusioncube /opt/huawei/fce/apps/platform/webapps/SSOSvr/WEB-INF/classes/services/Nexent-10000001.json -``` - ### Northbound Interface Configuration (NORTHBOUND_EXTERNAL_URL) If you need to use any of the following features, configure the `NORTHBOUND_EXTERNAL_URL` environment variable: diff --git a/doc/docs/en/quick-start/kubernetes-installation.md b/doc/docs/en/quick-start/kubernetes-installation.md index a10873c7c..8253c411f 100644 --- a/doc/docs/en/quick-start/kubernetes-installation.md +++ b/doc/docs/en/quick-start/kubernetes-installation.md @@ -291,122 +291,6 @@ Provider callback URLs: For local NodePort, a GitHub callback example is `http://localhost:30000/api/user/oauth/callback?provider=github`. In production, use a public HTTPS domain and register the exact same URL in the OAuth provider console. -### CAS Login Configuration - -CAS SSO does not require the `supabase` component. Set `nexent-common.config.cas.callbackBaseUrl` to the browser-accessible Nexent Web URL without a trailing `/`. `nexent-common.config.cas.serverUrl` is the CAS Server root URL and should also not include a trailing `/`. - -Kubernetes writes CAS settings into backend environment variables through `nexent-common` `config.cas.*` values: - -```bash -helm upgrade --install nexent nexent \ - --namespace nexent --create-namespace \ - --set nexent-common.config.cas.enabled=true \ - --set nexent-common.config.cas.serverUrl=https://cas.example.com/cas \ - --set nexent-common.config.cas.callbackBaseUrl=https://nexent.example.com \ - --set nexent-common.config.cas.loginMode=force \ - --set nexent-common.config.cas.logoutUrl=/logout -``` - -Configurable CAS values: - -| Value | Environment variable | Description | -|-------|----------------------|-------------| -| `nexent-common.config.cas.enabled` | `CAS_ENABLED` | Enables CAS | -| `nexent-common.config.cas.serverUrl` | `CAS_SERVER_URL` | CAS Server root URL | -| `nexent-common.config.cas.validatePath` | `CAS_VALIDATE_PATH` | serviceValidate path, default `/p3/serviceValidate` | -| `nexent-common.config.cas.callbackBaseUrl` | `CAS_CALLBACK_BASE_URL` | Web entry URL; CAS callback paths are appended automatically | -| `nexent-common.config.cas.loginMode` | `CAS_LOGIN_MODE` | `disabled`, `button`, or `force` | -| `nexent-common.config.cas.userAttribute` | `CAS_USER_ATTRIBUTE` | User identifier attribute. Empty means use `` | -| `nexent-common.config.cas.emailAttribute` | `CAS_EMAIL_ATTRIBUTE` | Email attribute | -| `nexent-common.config.cas.roleAttribute` | `CAS_ROLE_ATTRIBUTE` | Role attribute | -| `nexent-common.config.cas.tenantAttribute` | `CAS_TENANT_ATTRIBUTE` | Tenant attribute | -| `nexent-common.config.cas.roleMapJson` | `CAS_ROLE_MAP_JSON` | JSON mapping from CAS roles to Nexent roles | -| `nexent-common.config.cas.sessionMaxAgeSeconds` | `CAS_SESSION_MAX_AGE_SECONDS` | Maximum local CAS session lifetime | -| `nexent-common.config.cas.localSessionMaxAgeSeconds` | `LOCAL_SESSION_MAX_AGE_SECONDS` | Nexent local session lifetime | -| `nexent-common.config.cas.renewBeforeSeconds` | `CAS_RENEW_BEFORE_SECONDS` | Trigger silent renewal within this many seconds before expiry | -| `nexent-common.config.cas.renewTimeoutSeconds` | `CAS_RENEW_TIMEOUT_SECONDS` | Silent renewal timeout | -| `nexent-common.config.cas.syntheticEmailDomain` | `CAS_SYNTHETIC_EMAIL_DOMAIN` | Domain used when CAS does not return an email | -| `nexent-common.config.cas.logoutUrl` | `CAS_LOGOUT_URL` | CAS logout URL. Empty means Nexent logout will not call the CAS Server logout endpoint | -| `nexent-common.config.cas.sslVerify` | `CAS_SSL_VERIFY` | Whether to verify CAS Server TLS certificates | -| `nexent-common.config.cas.caBundle` | `CAS_CA_BUNDLE` | Custom CA bundle path | - -Common CAS URLs: - -| Purpose | URL | -|---------|-----| -| Nexent login entry | `{CAS_CALLBACK_BASE_URL}/api/user/cas/login?redirect=/` | -| CAS service callback | `{CAS_CALLBACK_BASE_URL}/api/user/cas/callback` | -| CAS silent renewal callback | `{CAS_CALLBACK_BASE_URL}/api/user/cas/renew_callback` | -| CAS single logout callback | `POST {CAS_CALLBACK_BASE_URL}/api/user/cas/logout_callback` | - -For Apereo CAS JSON Service Registry, create a service registration file such as `Nexent-10001.json` in the service registry directory configured by your CAS deployment. The `id` must be globally unique. This is a local NodePort example: - -```json -{ - "@class": "org.apereo.cas.services.RegexRegisteredService", - "serviceId": "http://localhost:30000.*", - "name": "Nexent CAS Client", - "id": 10001, - "description": "Nexent CAS SSO client", - "evaluationOrder": 1, - "logoutType": "BACK_CHANNEL", - "logoutUrl": "http://localhost:30000/api/user/cas/logout_callback" -} -``` - -In production, keep `CAS_SSL_VERIFY=true`; for self-signed certificates, prefer `CAS_CA_BUNDLE` and only use `CAS_SSL_VERIFY=false` for local testing. - -#### CAS Integration with ModelEngine - -When integrating with ModelEngine through the CAS protocol, use a values file to configure Nexent. This avoids complex command-line escaping for `CAS_ROLE_MAP_JSON`. - -Create `cas-modelengine-values.yaml`: - -```yaml -nexent-common: - config: - cas: - enabled: true - serverUrl: "https://:5443/SSOSvr" - validatePath: "/p3/serviceValidate" - callbackBaseUrl: "http://:30000" - loginMode: "force" - userAttribute: "userName" - emailAttribute: "email" - roleAttribute: "userType" - tenantAttribute: "tenant_id" - roleMapJson: '{"1":"ADMIN","3":"DEV"}' - sessionMaxAgeSeconds: 3600 - localSessionMaxAgeSeconds: 3600 - renewBeforeSeconds: 300 - renewTimeoutSeconds: 10 - syntheticEmailDomain: "cas.local" - logoutUrl: "/logout?service=http://:30000" - sslVerify: false - caBundle: "" -``` - -You also need to add a CAS client service registration file in the OMS container. Use the following steps as a reference: - -```bash -# Create the registration file, paste the JSON content into it, and save it. -vim Nexent-10000001.json -{ - "@class": "org.apereo.cas.services.CasRegisteredService", - "serviceId": "http://:30000.*", - "name": "Nexent CAS Client", - "id": 1000001, - "description": "Nexent CAS SSO client", - "evaluationOrder": 1, - "logoutType": "BACK_CHANNEL", - "logoutUrl": "http://:30000/api/user/cas/logout_callback" -} - -# Run the following command to copy the registration file into the container. -kubectl cp Nexent-10000001.json model-engine/$(kubectl get pods -n model-engine -l app=oms --no-headers | awk '{print $1}'):/opt/huawei/fce/apps/platform/webapps/SSOSvr/WEB-INF/classes/services/Nexent-10000001.json -kubectl exec -i -n model-engine $(kubectl get pods -n model-engine -l app=oms --no-headers | awk '{print $1}') -- chown tomcat:fusioncube /opt/huawei/fce/apps/platform/webapps/SSOSvr/WEB-INF/classes/services/Nexent-10000001.json -``` - ## 🔍 Troubleshooting ### Check Pod Status diff --git a/doc/docs/en/user-guide/agent-development.md b/doc/docs/en/user-guide/agent-development.md index 8e6b47d4f..7637cd620 100644 --- a/doc/docs/en/user-guide/agent-development.md +++ b/doc/docs/en/user-guide/agent-development.md @@ -111,18 +111,6 @@ In the External A2A Agent list, you can view and manage all discovered external > - Batch integrate all agents from the same service registry through Nacos discovery > - Configure protocols to meet the requirements of different agent service providers -###### Integrate [DataAgent](https://gitcode.com/datagallery/dataagent) A2A Agent via URL - -1. Refer to the [DataAgent documentation](https://gitcode.com/datagallery/dataagent#%F0%9F%8C%90-a2a-10-%E6%9C%8D%E5%8A%A1%E6%A8%A1%E5%BC%8F) and start DataAgent in A2A service mode. - > Nexent does not currently support agents that require authentication. Do not set `auth-token` when starting DataAgent. - -
- -
- -2. Refer to [Discover Agent via URL](#discover-agent-via-url) to integrate the agent. The URL is `http://:9999/.well-known/agent-card.json`. -3. Refer to [Manage Discovered External Agents](#manage-discovered-external-agents) to configure the invocation protocol, and select HTTP + JSON for integration. - ### 🛠️ Select Agent Tools Agents can use various tools to complete tasks, such as knowledge base search, file parsing, image parsing, email sending/receiving, file management, and other local tools. They can also integrate third-party MCP tools or custom tools. diff --git a/doc/docs/en/user-guide/assets/agent-development/dataagent_deploy.png b/doc/docs/en/user-guide/assets/agent-development/dataagent_deploy.png deleted file mode 100644 index 46fa9fde3..000000000 Binary files a/doc/docs/en/user-guide/assets/agent-development/dataagent_deploy.png and /dev/null differ diff --git a/doc/docs/zh/quick-start/installation.md b/doc/docs/zh/quick-start/installation.md index 6d3538b90..871cae0cc 100644 --- a/doc/docs/zh/quick-start/installation.md +++ b/doc/docs/zh/quick-start/installation.md @@ -269,111 +269,6 @@ Provider 启用规则： 本地默认回调示例为 `http://localhost:3000/api/user/oauth/callback?provider=github`。生产环境应改为公网 HTTPS 域名，例如 `https://nexent.example.com/api/user/oauth/callback?provider=github`，并在 OAuth provider 控制台中登记相同地址。 -### CAS 登录配置 - -CAS SSO 不依赖 `supabase`。启用 CAS 时，请将 `CAS_CALLBACK_BASE_URL` 设置为浏览器可访问的 Nexent Web 地址，且不要带结尾 `/`。`CAS_SERVER_URL` 是 CAS Server 根地址，也不要带结尾 `/`。 - -Docker 部署在 `docker/.env` 中配置 CAS： - -```bash -CAS_ENABLED=true -CAS_SERVER_URL=http://localhost:8080/cas -CAS_VALIDATE_PATH=/p3/serviceValidate -CAS_CALLBACK_BASE_URL=http://localhost:3000 - -# disabled: 禁用 CAS 登录入口和自动跳转 -# button: 在登录页显示 CAS 登录按钮 -# force: 未登录访问 Nexent 时自动跳转到 CAS -CAS_LOGIN_MODE=force - -# 为空时使用 ；填写 userName 时从 取用户标识 -CAS_USER_ATTRIBUTE= -CAS_EMAIL_ATTRIBUTE=email -CAS_ROLE_ATTRIBUTE=role -CAS_TENANT_ATTRIBUTE=tenant_id -CAS_ROLE_MAP_JSON={"cas-admin":"ADMIN","cas-user":"USER"} -CAS_SESSION_MAX_AGE_SECONDS=3600 -LOCAL_SESSION_MAX_AGE_SECONDS=3600 -CAS_RENEW_BEFORE_SECONDS=300 -CAS_RENEW_TIMEOUT_SECONDS=10 -CAS_SYNTHETIC_EMAIL_DOMAIN=cas.local - -# 为空时 Nexent 主动退出不会调用 CAS Server 登出接口。 -# 可配置为 /logout，系统会基于 CAS_SERVER_URL 拼接。 -CAS_LOGOUT_URL=/logout -CAS_SSL_VERIFY=true -CAS_CA_BUNDLE= -``` - -常用 CAS 地址： - -| 用途 | 地址 | -|------|------| -| Nexent 登录入口 | `{CAS_CALLBACK_BASE_URL}/api/user/cas/login?redirect=/` | -| CAS service 回调 | `{CAS_CALLBACK_BASE_URL}/api/user/cas/callback` | -| CAS 无感续期回调 | `{CAS_CALLBACK_BASE_URL}/api/user/cas/renew_callback` | -| CAS 单点登出回调 | `POST {CAS_CALLBACK_BASE_URL}/api/user/cas/logout_callback` | - -Apereo CAS 使用 JSON Service Registry 时，可以新增一个服务注册文件，例如 `Nexent-10001.json`。文件需要放到 CAS 部署配置的 service registry 目录中，`id` 必须全局唯一。下面是本地 Docker 示例： - -```json -{ - "@class": "org.apereo.cas.services.RegexRegisteredService", - "serviceId": "http://localhost:3000.*", - "name": "Nexent CAS Client", - "id": 10001, - "description": "Nexent CAS SSO client", - "evaluationOrder": 1, - "logoutType": "BACK_CHANNEL", - "logoutUrl": "http://localhost:3000/api/user/cas/logout_callback" -} -``` - -生产环境建议保持 `CAS_SSL_VERIFY=true`；自签名证书优先配置 `CAS_CA_BUNDLE`，仅本地验证时再临时设置 `CAS_SSL_VERIFY=false`。 - -#### CAS对接ModelEngine -当使用CAS协议对接ModelEngine时，可以使用如下配置部署Nexent： -```bash -CAS_ENABLED=true -CAS_SERVER_URL=https://:5443/SSOSvr -CAS_VALIDATE_PATH=/p3/serviceValidate -CAS_CALLBACK_BASE_URL=http://:3000 -CAS_LOGIN_MODE=force -CAS_USER_ATTRIBUTE=userName -CAS_EMAIL_ATTRIBUTE=email -CAS_ROLE_ATTRIBUTE=userType -CAS_TENANT_ATTRIBUTE=tenant_id -CAS_ROLE_MAP_JSON={"1":"ADMIN","3":"DEV"} -CAS_SESSION_MAX_AGE_SECONDS=3600 -LOCAL_SESSION_MAX_AGE_SECONDS=3600 -CAS_RENEW_BEFORE_SECONDS=300 -CAS_RENEW_TIMEOUT_SECONDS=10 -CAS_SYNTHETIC_EMAIL_DOMAIN=cas.local -CAS_LOGOUT_URL=/logout?service=http://:3000 -CAS_SSL_VERIFY=false -CAS_CA_BUNDLE= -``` - -同时，需要进入oms容器添加cas client的注册配置文件，参考如下步骤： -```bash -# 创建注册配置文件，将json部分输入文件并保存 -vim Nexent-10000001.json -{ - "@class": "org.apereo.cas.services.CasRegisteredService", - "serviceId": "http://:3000.*", - "name": "Nexent CAS Client", - "id": 1000001, - "description": "Nexent CAS SSO client", - "evaluationOrder": 1, - "logoutType": "BACK_CHANNEL", - "logoutUrl": "http://:3000/api/user/cas/logout_callback" -} - -# 执行如下命令，将配置文件拷贝到容器中 -kubectl cp Nexent-10000001.json model-engine/$(kubectl get pods -n model-engine -l app=oms --no-headers | awk '{print $1}'):/opt/huawei/fce/apps/platform/webapps/SSOSvr/WEB-INF/classes/services/Nexent-10000001.json -kubectl exec -i -n model-engine $(kubectl get pods -n model-engine -l app=oms --no-headers | awk '{print $1}') -- chown tomcat:fusioncube /opt/huawei/fce/apps/platform/webapps/SSOSvr/WEB-INF/classes/services/Nexent-10000001.json -``` - ### 北向接口配置 (NORTHBOUND_EXTERNAL_URL) 如果您需要使用以下功能，需要配置 `NORTHBOUND_EXTERNAL_URL` 环境变量： diff --git a/doc/docs/zh/quick-start/kubernetes-installation.md b/doc/docs/zh/quick-start/kubernetes-installation.md index 7229f1ea8..47d2af816 100644 --- a/doc/docs/zh/quick-start/kubernetes-installation.md +++ b/doc/docs/zh/quick-start/kubernetes-installation.md @@ -291,122 +291,6 @@ Provider 回调地址： 本地 NodePort 默认回调示例为 `http://localhost:30000/api/user/oauth/callback?provider=github`。生产环境应改为公网 HTTPS 域名，并在 OAuth provider 控制台中登记相同地址。 -### CAS 登录配置 - -CAS SSO 不依赖 `supabase`。启用 CAS 时，请将 `nexent-common.config.cas.callbackBaseUrl` 设置为浏览器可访问的 Nexent Web 地址，且不要带结尾 `/`。`nexent-common.config.cas.serverUrl` 是 CAS Server 根地址，也不要带结尾 `/`。 - -Kubernetes 部署通过 `nexent-common` 的 `config.cas.*` values 写入后端环境变量： - -```bash -helm upgrade --install nexent nexent \ - --namespace nexent --create-namespace \ - --set nexent-common.config.cas.enabled=true \ - --set nexent-common.config.cas.serverUrl=https://cas.example.com/cas \ - --set nexent-common.config.cas.callbackBaseUrl=https://nexent.example.com \ - --set nexent-common.config.cas.loginMode=force \ - --set nexent-common.config.cas.logoutUrl=/logout -``` - -可配置的 CAS values： - -| Values | 对应环境变量 | 说明 | -|--------|--------------|------| -| `nexent-common.config.cas.enabled` | `CAS_ENABLED` | 是否启用 CAS | -| `nexent-common.config.cas.serverUrl` | `CAS_SERVER_URL` | CAS Server 根地址 | -| `nexent-common.config.cas.validatePath` | `CAS_VALIDATE_PATH` | serviceValidate 路径，默认 `/p3/serviceValidate` | -| `nexent-common.config.cas.callbackBaseUrl` | `CAS_CALLBACK_BASE_URL` | Web 入口地址，CAS 回调路径会自动拼接 | -| `nexent-common.config.cas.loginMode` | `CAS_LOGIN_MODE` | `disabled`、`button` 或 `force` | -| `nexent-common.config.cas.userAttribute` | `CAS_USER_ATTRIBUTE` | 用户标识属性。为空时使用 `` | -| `nexent-common.config.cas.emailAttribute` | `CAS_EMAIL_ATTRIBUTE` | 邮箱属性 | -| `nexent-common.config.cas.roleAttribute` | `CAS_ROLE_ATTRIBUTE` | 角色属性 | -| `nexent-common.config.cas.tenantAttribute` | `CAS_TENANT_ATTRIBUTE` | 租户属性 | -| `nexent-common.config.cas.roleMapJson` | `CAS_ROLE_MAP_JSON` | CAS 角色到 Nexent 角色的 JSON 映射 | -| `nexent-common.config.cas.sessionMaxAgeSeconds` | `CAS_SESSION_MAX_AGE_SECONDS` | CAS 本地会话最长有效期 | -| `nexent-common.config.cas.localSessionMaxAgeSeconds` | `LOCAL_SESSION_MAX_AGE_SECONDS` | Nexent 本地会话有效期 | -| `nexent-common.config.cas.renewBeforeSeconds` | `CAS_RENEW_BEFORE_SECONDS` | 距离过期多少秒内触发无感续期 | -| `nexent-common.config.cas.renewTimeoutSeconds` | `CAS_RENEW_TIMEOUT_SECONDS` | 无感续期等待超时时间 | -| `nexent-common.config.cas.syntheticEmailDomain` | `CAS_SYNTHETIC_EMAIL_DOMAIN` | CAS 未返回邮箱时生成邮箱使用的域名 | -| `nexent-common.config.cas.logoutUrl` | `CAS_LOGOUT_URL` | CAS 登出地址。为空时 Nexent 主动退出不调用 CAS Server 登出接口 | -| `nexent-common.config.cas.sslVerify` | `CAS_SSL_VERIFY` | 访问 CAS Server 时是否校验证书 | -| `nexent-common.config.cas.caBundle` | `CAS_CA_BUNDLE` | 自定义 CA bundle 路径 | - -常用 CAS 地址： - -| 用途 | 地址 | -|------|------| -| Nexent 登录入口 | `{CAS_CALLBACK_BASE_URL}/api/user/cas/login?redirect=/` | -| CAS service 回调 | `{CAS_CALLBACK_BASE_URL}/api/user/cas/callback` | -| CAS 无感续期回调 | `{CAS_CALLBACK_BASE_URL}/api/user/cas/renew_callback` | -| CAS 单点登出回调 | `POST {CAS_CALLBACK_BASE_URL}/api/user/cas/logout_callback` | - -Apereo CAS 使用 JSON Service Registry 时，可以新增一个服务注册文件，例如 `Nexent-10001.json`。文件需要放到 CAS 部署配置的 service registry 目录中，`id` 必须全局唯一。本地 NodePort 示例： - -```json -{ - "@class": "org.apereo.cas.services.RegexRegisteredService", - "serviceId": "http://localhost:30000.*", - "name": "Nexent CAS Client", - "id": 10001, - "description": "Nexent CAS SSO client", - "evaluationOrder": 1, - "logoutType": "BACK_CHANNEL", - "logoutUrl": "http://localhost:30000/api/user/cas/logout_callback" -} -``` - -生产环境建议保持 `CAS_SSL_VERIFY=true`；自签名证书优先配置 `CAS_CA_BUNDLE`，仅本地验证时再临时设置 `CAS_SSL_VERIFY=false`。 - -#### CAS 对接 ModelEngine - -当使用 CAS 协议对接 ModelEngine 时，建议通过 values 文件配置 Nexent，避免 `CAS_ROLE_MAP_JSON` 在命令行中转义复杂。 - -创建 `cas-modelengine-values.yaml`： - -```yaml -nexent-common: - config: - cas: - enabled: true - serverUrl: "https://:5443/SSOSvr" - validatePath: "/p3/serviceValidate" - callbackBaseUrl: "http://:30000" - loginMode: "force" - userAttribute: "userName" - emailAttribute: "email" - roleAttribute: "userType" - tenantAttribute: "tenant_id" - roleMapJson: '{"1":"ADMIN","3":"DEV"}' - sessionMaxAgeSeconds: 3600 - localSessionMaxAgeSeconds: 3600 - renewBeforeSeconds: 300 - renewTimeoutSeconds: 10 - syntheticEmailDomain: "cas.local" - logoutUrl: "/logout?service=http://:30000" - sslVerify: false - caBundle: "" -``` - -同时，需要进入 OMS 容器添加 CAS client 的注册配置文件，参考如下步骤： - -```bash -# 创建注册配置文件，将 JSON 部分输入文件并保存 -vim Nexent-10000001.json -{ - "@class": "org.apereo.cas.services.CasRegisteredService", - "serviceId": "http://:30000.*", - "name": "Nexent CAS Client", - "id": 1000001, - "description": "Nexent CAS SSO client", - "evaluationOrder": 1, - "logoutType": "BACK_CHANNEL", - "logoutUrl": "http://:30000/api/user/cas/logout_callback" -} - -# 执行如下命令，将配置文件拷贝到容器中 -kubectl cp Nexent-10000001.json model-engine/$(kubectl get pods -n model-engine -l app=oms --no-headers | awk '{print $1}'):/opt/huawei/fce/apps/platform/webapps/SSOSvr/WEB-INF/classes/services/Nexent-10000001.json -kubectl exec -i -n model-engine $(kubectl get pods -n model-engine -l app=oms --no-headers | awk '{print $1}') -- chown tomcat:fusioncube /opt/huawei/fce/apps/platform/webapps/SSOSvr/WEB-INF/classes/services/Nexent-10000001.json -``` - ## 🔍 故障排查 ### 查看 Pod 状态 diff --git a/doc/docs/zh/sdk/vector-database.md b/doc/docs/zh/sdk/vector-database.md index b940400fd..940af9c33 100644 --- a/doc/docs/zh/sdk/vector-database.md +++ b/doc/docs/zh/sdk/vector-database.md @@ -579,11 +579,7 @@ python -m nexent.service.vectordatabase_service - 参数: - `index_name`: 索引名称 (路径参数) - `path_or_url`: 文档路径或URL (查询参数) - - `scope`: 删除范围 (查询参数，默认 `full`) - - `source_only`: 仅删除 MinIO 源文件，保留 ES 中的切片与向量（检索仍可用，预览不可用） - - `full`: 删除 ES 文档、MinIO 源文件，并清理相关 Redis 任务记录 - - 返回示例 (`source_only`): `{"status": "success", "scope": "source_only", "deleted_es_count": 0, "deleted_minio": true, "source_available": false}` - - 返回示例 (`full`): `{"status": "success", "scope": "full", "deleted_es_count": 5, "deleted_minio": true}` + - 返回示例: `{"status": "success", "deleted_count": 1}` #### 搜索操作 @@ -732,11 +728,8 @@ curl -X POST "http://localhost:8000/indices/search/hybrid" \ "weight_accurate": 0.3 }' -# 删除源文件（保留索引） -curl -X DELETE "http://localhost:8000/indices/my_documents/documents?path_or_url=knowledge_base/doc1.pdf&scope=source_only" - -# 从知识库彻底移除文档 -curl -X DELETE "http://localhost:8000/indices/my_documents/documents?path_or_url=knowledge_base/doc1.pdf&scope=full" +# 删除文档 +curl -X DELETE "http://localhost:8000/indices/my_documents/documents?path_or_url=https://example.com/doc1" # 创建索引 curl -X POST "http://localhost:8000/indices/my_documents" diff --git a/doc/docs/zh/user-guide/agent-development.md b/doc/docs/zh/user-guide/agent-development.md index 40805aeea..3edf31de7 100644 --- a/doc/docs/zh/user-guide/agent-development.md +++ b/doc/docs/zh/user-guide/agent-development.md @@ -113,17 +113,6 @@ Nexent 支持通过 A2A 协议与第三方 Agent 进行通信。您可以通过 > - 通过 Nacos 发现批量接入同一服务注册中心的所有 Agent > - 配置协议以兼容不同 Agent 服务提供商的要求 - -###### 通过URL对接[DataAgent](https://gitcode.com/datagallery/dataagent) A2A Agent -1. 参考[DataAgent文档](https://gitcode.com/datagallery/dataagent#%F0%9F%8C%90-a2a-10-%E6%9C%8D%E5%8A%A1%E6%A8%A1%E5%BC%8F)以A2A服务模式启动DataAgent - >当前Nexent不支持带认证的agent，启动DataAgent时请勿设置auth-token -
- -
- -2. 参考[通过 URL 发现 Agent](#通过-url-发现-agent)接入agent，url为http://\:9999/.well-known/agent-card.json -3. 参考[管理已发现的外部 Agent](#管理已发现的外部-agent)配置调用协议，选择HTTP+JSON方式接入 - ### 🛠️ 选择智能体的工具 智能体可以使用各种工具来完成任务，如知识库检索、文件解析、图片解析、收发邮件、文件管理等本地工具，也可接入第三方 MCP 工具，或自定义工具。 diff --git a/doc/docs/zh/user-guide/assets/agent-development/dataagent_deploy.png b/doc/docs/zh/user-guide/assets/agent-development/dataagent_deploy.png deleted file mode 100644 index 46fa9fde3..000000000 Binary files a/doc/docs/zh/user-guide/assets/agent-development/dataagent_deploy.png and /dev/null differ diff --git a/doc/procedural-memory-verification.md b/doc/procedural-memory-verification.md deleted file mode 100644 index ea9f53290..000000000 --- a/doc/procedural-memory-verification.md +++ /dev/null @@ -1,315 +0,0 @@ -# Procedural Memory Verification Report - -## Summary -**Status: ⚠️ FULLY SUPPORTED but REQUIRES OPTIONAL DEPENDENCY** - -Procedural memory is a fully implemented feature in mem0ai version 0.1.117, **BUT it requires `langchain-core` to be installed separately**. Without this dependency, the feature will fail at runtime. - ---- - -## ⚠️ CRITICAL FINDING: Optional Dependency Required - -**Your colleague is partially correct.** The procedural memory code is NOT empty (it's 50 lines of real implementation), but it has a critical dependency issue: - -### The Problem - -The `_create_procedural_memory()` method contains: - -```python -try: - from langchain_core.messages.utils import convert_to_messages -except Exception: - logger.error( - "Import error while loading langchain-core. " - "Please install 'langchain-core' to use procedural memory." - ) - raise # ← Fails here if langchain-core not installed -``` - -### Reality Check - -| Aspect | Status | -|--------|--------| -| Code exists? | ✅ Yes, 50 lines of real implementation | -| Code is empty/stub? | ❌ No, it's fully implemented | -| Works out of the box? | ❌ **NO** - requires `langchain-core` package | -| Documented requirement? | ⚠️ Only in error message, not in main docs | - -### Why Your Colleague Thought It Was Empty - -1. They called `memory.add(..., memory_type="procedural_memory")` -2. Got `ImportError: No module named 'langchain_core'` -3. Saw the error and concluded "it doesn't work" or "it's empty" -4. This is understandable - the feature exists but is **disabled by default** - ---- - -## Verification Results - -### 1. API Support ✅ -The `memory_type` parameter is available in both `AsyncMemory.add()` and `Memory.add()`: - -```python -async def add( - self, - messages, - *, - user_id: Optional[str] = None, - agent_id: Optional[str] = None, - run_id: Optional[str] = None, - metadata: Optional[Dict[str, Any]] = None, - infer: bool = True, - memory_type: Optional[str] = None, # ✅ SUPPORTED - prompt: Optional[str] = None, - llm=None -) -``` - -### 2. MemoryType Enum ✅ -Located in `mem0.configs.enums.MemoryType`: - -```python -class MemoryType(Enum): - SEMANTIC = "semantic_memory" - EPISODIC = "episodic_memory" - PROCEDURAL = "procedural_memory" # ✅ AVAILABLE -``` - -### 3. Implementation ✅ -The `_create_procedural_memory()` method exists in both `AsyncMemory` and `Memory` classes: - -**AsyncMemory signature:** -```python -async def _create_procedural_memory( - self, - messages, - metadata=None, - llm=None, - prompt=None -) -``` - -**Memory (sync) signature:** -```python -def _create_procedural_memory( - self, - messages, - metadata=None, - prompt=None -) -``` - -### 4. Validation Logic ✅ -The `add()` method validates `memory_type` and enforces constraints: - -```python -# Only "procedural_memory" is accepted -if memory_type is not None and memory_type != MemoryType.PROCEDURAL.value: - raise ValueError( - f"Invalid 'memory_type'. Please pass {MemoryType.PROCEDURAL.value} " - "to create procedural memories." - ) - -# agent_id is REQUIRED for procedural memory -if agent_id is not None and memory_type == MemoryType.PROCEDURAL.value: - results = await self._create_procedural_memory( - messages, metadata=processed_metadata, prompt=prompt, llm=llm - ) - return results -``` - -### 5. System Prompt ✅ -A comprehensive 5,100-character system prompt exists in `mem0.configs.prompts.PROCEDURAL_MEMORY_SYSTEM_PROMPT`: - -**Purpose:** Records and preserves complete interaction history between human and AI agent - -**Structure:** -- Overview (Global Metadata) - - Task Objective - - Progress Status -- Sequential Agent Actions (Numbered Steps) - - Agent Action - - Action Result (Mandatory, Unmodified) - - Embedded Metadata (Key Findings, Navigation History, Errors, Current Context) - -**Key Guidelines:** -1. Preserve every output verbatim -2. Maintain chronological order -3. Include exact data (URLs, element indexes, error messages, JSON responses) -4. Output only the structured summary - ---- - -## Usage Example - -```python -from mem0 import AsyncMemory - -# Initialize memory -memory = await AsyncMemory.from_config(config) - -# Create procedural memory -messages = [ - {"role": "user", "content": "Search for AI news"}, - {"role": "assistant", "content": "I'll search for recent AI news..."}, - # ... more conversation history -] - -result = await memory.add( - messages=messages, - user_id="user_123", - agent_id="research_agent", # ⚠️ REQUIRED for procedural memory - memory_type="procedural_memory", - metadata={ - "task": "AI news research", - "session_id": "session_456" - } -) - -# Result format: -# { -# "results": [ -# { -# "id": "memory_id_here", -# "memory": "## Summary of the agent's execution history...", -# "event": "ADD" -# } -# ] -# } -``` - ---- - -## Requirements & Constraints - -### Required Parameters -- ✅ `agent_id`: **MUST** be provided when using `memory_type="procedural_memory"` -- ✅ `metadata`: **MUST** be provided (cannot be None) -- ✅ `messages`: List of conversation messages to summarize - -### Optional Parameters -- `prompt`: Custom prompt to override default `PROCEDURAL_MEMORY_SYSTEM_PROMPT` -- `llm`: Custom LangChain ChatModel (async version only) - -### Validation Rules -1. `memory_type` must be exactly `"procedural_memory"` (or None) -2. If `memory_type="procedural_memory"` is set, `agent_id` must be provided -3. `metadata` cannot be None for procedural memories - ---- - -## Implementation Details - -### How It Works -1. **Validation**: Checks `memory_type` and required parameters -2. **Prompt Construction**: Uses default or custom system prompt -3. **LLM Summarization**: Calls LLM to generate comprehensive execution summary -4. **Embedding**: Generates embedding for the summary -5. **Storage**: Stores in vector database with `metadata["memory_type"] = "procedural_memory"` -6. **Return**: Returns memory ID and summary text - -### Async vs Sync -- **AsyncMemory**: Supports custom LangChain `llm` parameter -- **Memory**: Uses internal LLM from config only - ---- - -## Integration with Nexent - -### Current Status -The Nexent codebase does **NOT** currently use procedural memory. The `memory_type` parameter is not passed in any `add_memory()` calls. - -### Recommended Integration Points - -1. **Agent Service** (`backend/services/agent_service.py`): - - Detect when agent completes a multi-step task - - Call `add_memory_in_levels()` with `memory_type="procedural_memory"` - - Pass the full conversation history as messages - -2. **Memory Service** (`sdk/nexent/memory/memory_service.py`): - - Add `memory_type` parameter to `add_memory()` and `add_memory_in_levels()` - - Pass through to mem0's `add()` method - -3. **Agent Run Info** (`sdk/nexent/core/agents/agent_model.py`): - - Add `memory_type` field to track if current run should create procedural memory - -### Example Integration - -```python -# In agent_service.py, after agent completes a complex task -if task_complexity >= threshold: # Your logic here - await add_memory_in_levels( - messages=conversation_history, - memory_config=memory_ctx.memory_config, - tenant_id=memory_ctx.tenant_id, - user_id=memory_ctx.user_id, - agent_id=memory_ctx.agent_id, - memory_levels=["agent", "user_agent"], - memory_type="procedural_memory", # ✅ NEW PARAMETER - metadata={ - "task_type": "complex_research", - "duration_seconds": duration, - "steps_completed": step_count - } - ) -``` - ---- - -## Conclusion - -Procedural memory is a **fully functional feature** in mem0ai==0.1.117, **BUT it requires an optional dependency**. It provides: - -- ✅ Complete API support -- ✅ Comprehensive system prompt (5,100 characters) -- ✅ Proper validation and error handling -- ✅ Both sync and async implementations -- ✅ Integration with existing memory infrastructure -- ⚠️ **REQUIRES `langchain-core` package to be installed** - -### The Truth About "Empty Function" Claims - -**The code is NOT empty.** It's a 50-line implementation that: -1. Calls LLM to generate execution summary -2. Creates embeddings -3. Stores in vector database -4. Returns proper results - -**However, it fails at runtime** if `langchain-core` is not installed, which is why your colleague might have thought it was a no-op. - -### How to Enable - -**Option 1: Install the dependency** -```bash -pip install langchain-core -``` - -**Option 2: Add to Nexent's dependencies** -```toml -# In sdk/pyproject.toml -dependencies = [ - # ... existing deps ... - "langchain-core>=0.1.0", # Required for procedural memory -] -``` - -**Option 3: Make it optional with fallback** -```python -try: - result = await memory.add(..., memory_type="procedural_memory") -except ImportError as e: - if "langchain-core" in str(e): - logger.warning("Procedural memory requires langchain-core. Using regular memory.") - result = await memory.add(...) # Fallback - else: - raise -``` - -### Final Recommendation - -This feature **can be integrated into Nexent**, but you must: -1. Add `langchain-core` to dependencies, OR -2. Implement graceful fallback when dependency is missing, OR -3. Document it as an optional feature requiring extra installation - -Without addressing the dependency issue, procedural memory will fail at runtime despite having complete implementation code. diff --git a/docker/.env.example b/docker/.env.example index 3970efb95..c34300523 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -226,27 +226,3 @@ OAUTH_CALLBACK_BASE_URL=http://localhost:3000 # Asset owner role (opt-in; default false). Set true to enable ASSET_OWNER. ENABLE_ASSET_OWNER_ROLE=false - -# ===== CAS SSO Configuration ===== -CAS_ENABLED=false -CAS_SERVER_URL= -CAS_VALIDATE_PATH=/p3/serviceValidate -CAS_CALLBACK_BASE_URL=http://localhost:3000 -# Supported values: -# - disabled: disable CAS login entry and automatic CAS redirects. -# - button: show CAS as an optional login entry. -# - force: automatically redirect unauthenticated users to CAS login. -CAS_LOGIN_MODE=disabled -CAS_USER_ATTRIBUTE= -CAS_EMAIL_ATTRIBUTE=email -CAS_ROLE_ATTRIBUTE=role -CAS_TENANT_ATTRIBUTE=tenant_id -CAS_ROLE_MAP_JSON= -CAS_SESSION_MAX_AGE_SECONDS=3600 -LOCAL_SESSION_MAX_AGE_SECONDS=3600 -CAS_RENEW_BEFORE_SECONDS=300 -CAS_RENEW_TIMEOUT_SECONDS=10 -CAS_SYNTHETIC_EMAIL_DOMAIN=cas.local -CAS_LOGOUT_URL=/logout -CAS_SSL_VERIFY=true -CAS_CA_BUNDLE= diff --git a/docker/deploy.sh b/docker/deploy.sh index fbf3664b5..2069330d1 100755 --- a/docker/deploy.sh +++ b/docker/deploy.sh @@ -1367,7 +1367,7 @@ main_deploy() { echo "--------------------------------" echo "" - APP_VERSION="$(get_app_version)" + APP_VERSION="latest" if [ -z "$APP_VERSION" ]; then echo "❌ Failed to get app version, please check the backend/consts/const.py file" exit 1 diff --git a/docker/init.sql b/docker/init.sql index 046bdecf1..0668def01 100644 --- a/docker/init.sql +++ b/docker/init.sql @@ -230,7 +230,6 @@ CREATE TABLE IF NOT EXISTS "knowledge_record_t" ( "summary_frequency" varchar(10) COLLATE "pg_catalog"."default", "last_summary_time" timestamp(0), "last_doc_update_time" timestamp(0), - "preserve_source_file" boolean NOT NULL DEFAULT true, CONSTRAINT "knowledge_record_t_pk" PRIMARY KEY ("knowledge_id") ); ALTER TABLE "knowledge_record_t" OWNER TO "root"; @@ -252,7 +251,6 @@ COMMENT ON COLUMN "knowledge_record_t"."created_by" IS 'User who created the rec COMMENT ON COLUMN "knowledge_record_t"."summary_frequency" IS 'Auto-summary frequency: 1h, 3h, 6h, 1d, 1w, or NULL (disabled)'; COMMENT ON COLUMN "knowledge_record_t"."last_summary_time" IS 'Timestamp of last summary generation'; COMMENT ON COLUMN "knowledge_record_t"."last_doc_update_time" IS 'Timestamp of last document add/delete operation, used for auto-summary optimization to skip unnecessary summary regeneration'; -COMMENT ON COLUMN "knowledge_record_t"."preserve_source_file" IS 'Whether to preserve uploaded source documents after vectorization'; COMMENT ON COLUMN "knowledge_record_t"."updated_by" IS 'Last updater ID, audit field'; COMMENT ON COLUMN "knowledge_record_t"."created_by" IS 'Creator ID, audit field'; COMMENT ON TABLE "knowledge_record_t" IS 'Records knowledge base description and status information'; @@ -339,12 +337,9 @@ CREATE TABLE IF NOT EXISTS nexent.ag_tenant_agent_t ( is_new BOOLEAN DEFAULT FALSE, provide_run_summary BOOLEAN DEFAULT FALSE, enable_context_manager BOOLEAN DEFAULT FALSE, - verification_config JSONB, version_no INTEGER DEFAULT 0 NOT NULL, current_version_no INTEGER NULL, ingroup_permission VARCHAR(30), - greeting_message TEXT, - example_questions JSONB, create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, created_by VARCHAR(100), @@ -402,9 +397,6 @@ COMMENT ON COLUMN nexent.ag_tenant_agent_t.version_no IS 'Version number. 0 = dr COMMENT ON COLUMN nexent.ag_tenant_agent_t.current_version_no IS 'Current published version number. NULL means no version published yet'; COMMENT ON COLUMN nexent.ag_tenant_agent_t.ingroup_permission IS 'In-group permission: EDIT, READ_ONLY, PRIVATE'; COMMENT ON COLUMN nexent.ag_tenant_agent_t.enable_context_manager IS 'Whether to enable context management (compression) for this agent'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.verification_config IS 'Layered ReAct self-verification configuration'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.greeting_message IS 'Agent greeting message displayed on chat initial screen'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.example_questions IS 'List of example questions for starting a conversation with this agent'; -- Create index for is_new queries CREATE INDEX IF NOT EXISTS idx_ag_tenant_agent_t_is_new @@ -723,7 +715,6 @@ CREATE TABLE IF NOT EXISTS nexent.ag_agent_relation_t ( parent_agent_id INTEGER, tenant_id VARCHAR(100), version_no INTEGER DEFAULT 0 NOT NULL, - selected_agent_version_no INTEGER, create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, created_by VARCHAR(100), @@ -756,7 +747,6 @@ COMMENT ON COLUMN nexent.ag_agent_relation_t.selected_agent_id IS 'Selected agen COMMENT ON COLUMN nexent.ag_agent_relation_t.parent_agent_id IS 'Parent agent ID'; COMMENT ON COLUMN nexent.ag_agent_relation_t.tenant_id IS 'Tenant ID'; COMMENT ON COLUMN nexent.ag_agent_relation_t.version_no IS 'Version number. 0 = draft/editing state, >=1 = published snapshot'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.selected_agent_version_no IS 'Pinned version of selected_agent_id. NULL = use child current published version at runtime (legacy/draft).'; COMMENT ON COLUMN nexent.ag_agent_relation_t.create_time IS 'Creation time, audit field'; COMMENT ON COLUMN nexent.ag_agent_relation_t.update_time IS 'Update time, audit field'; COMMENT ON COLUMN nexent.ag_agent_relation_t.created_by IS 'Creator ID, audit field'; @@ -1270,6 +1260,7 @@ CREATE TABLE IF NOT EXISTS nexent.ag_skill_info_t ( config_schemas JSON, config_values JSON, source VARCHAR(30) DEFAULT 'official', + tenant_id VARCHAR(100), created_by VARCHAR(100), create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, updated_by VARCHAR(100), @@ -1909,31 +1900,3 @@ FOR EACH ROW EXECUTE FUNCTION update_mcp_community_record_update_time(); COMMENT ON TRIGGER update_mcp_community_record_update_time_trigger ON nexent.mcp_community_record_t IS 'Trigger to maintain update_time'; - -CREATE TABLE IF NOT EXISTS nexent.user_cas_session_t ( - cas_session_id SERIAL PRIMARY KEY, - session_id VARCHAR(100) NOT NULL UNIQUE, - user_id VARCHAR(100) NOT NULL, - cas_user_id VARCHAR(200) NOT NULL, - cas_session_index VARCHAR(500), - status VARCHAR(30) NOT NULL DEFAULT 'active', - expires_at TIMESTAMP NOT NULL, - revoked_at TIMESTAMP, - create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - -CREATE INDEX IF NOT EXISTS ix_user_cas_session_session_id - ON nexent.user_cas_session_t (session_id); -CREATE INDEX IF NOT EXISTS ix_user_cas_session_user_id - ON nexent.user_cas_session_t (user_id); -CREATE INDEX IF NOT EXISTS ix_user_cas_session_cas_user_id - ON nexent.user_cas_session_t (cas_user_id); - -COMMENT ON TABLE nexent.user_cas_session_t IS 'Server-side session records for CAS SSO login and logout synchronization'; -COMMENT ON COLUMN nexent.user_cas_session_t.session_id IS 'JWT sid claim for revocation checks'; -COMMENT ON COLUMN nexent.user_cas_session_t.cas_user_id IS 'User identifier returned by CAS'; -COMMENT ON COLUMN nexent.user_cas_session_t.cas_session_index IS 'CAS SessionIndex or service ticket'; diff --git a/docker/official-skills-zip/create-docx.zip b/docker/official-skills-zip/create-docx.zip deleted file mode 100644 index aa53e82b0..000000000 Binary files a/docker/official-skills-zip/create-docx.zip and /dev/null differ diff --git a/docker/sql/v2.2.0_0526_add_cas_session_t.sql b/docker/sql/v2.2.0_0526_add_cas_session_t.sql deleted file mode 100644 index 3f1aab4fa..000000000 --- a/docker/sql/v2.2.0_0526_add_cas_session_t.sql +++ /dev/null @@ -1,27 +0,0 @@ -CREATE TABLE IF NOT EXISTS nexent.user_cas_session_t ( - cas_session_id SERIAL PRIMARY KEY, - session_id VARCHAR(100) NOT NULL UNIQUE, - user_id VARCHAR(100) NOT NULL, - cas_user_id VARCHAR(200) NOT NULL, - cas_session_index VARCHAR(500), - status VARCHAR(30) NOT NULL DEFAULT 'active', - expires_at TIMESTAMP NOT NULL, - revoked_at TIMESTAMP, - create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - -CREATE INDEX IF NOT EXISTS ix_user_cas_session_session_id - ON nexent.user_cas_session_t (session_id); -CREATE INDEX IF NOT EXISTS ix_user_cas_session_user_id - ON nexent.user_cas_session_t (user_id); -CREATE INDEX IF NOT EXISTS ix_user_cas_session_cas_user_id - ON nexent.user_cas_session_t (cas_user_id); - -COMMENT ON TABLE nexent.user_cas_session_t IS 'Server-side session records for CAS SSO login and logout synchronization'; -COMMENT ON COLUMN nexent.user_cas_session_t.session_id IS 'JWT sid claim for revocation checks'; -COMMENT ON COLUMN nexent.user_cas_session_t.cas_user_id IS 'User identifier returned by CAS'; -COMMENT ON COLUMN nexent.user_cas_session_t.cas_session_index IS 'CAS SessionIndex or service ticket'; diff --git a/docker/sql/v2.2.1_0601_add_agent_verification_config.sql b/docker/sql/v2.2.1_0601_add_agent_verification_config.sql deleted file mode 100644 index d3882e1e2..000000000 --- a/docker/sql/v2.2.1_0601_add_agent_verification_config.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Migration: Add layered ReAct self-verification config to agents --- Description: Stores per-agent verification controls for step-level and final-answer validation. - -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS verification_config JSONB; - -COMMENT ON COLUMN nexent.ag_tenant_agent_t.verification_config IS 'Layered ReAct self-verification configuration'; diff --git a/docker/sql/v2.2.1_0601_add_preserve_source_file_to_knowledge_record_t.sql b/docker/sql/v2.2.1_0601_add_preserve_source_file_to_knowledge_record_t.sql deleted file mode 100644 index 30b588a51..000000000 --- a/docker/sql/v2.2.1_0601_add_preserve_source_file_to_knowledge_record_t.sql +++ /dev/null @@ -1,8 +0,0 @@ --- Migration: Add preserve_source_file to knowledge_record_t table --- Date: 2026-06-01 --- Description: Whether to preserve uploaded source documents after vectorization (default: true) - -ALTER TABLE nexent.knowledge_record_t -ADD COLUMN IF NOT EXISTS preserve_source_file BOOLEAN NOT NULL DEFAULT true; - -COMMENT ON COLUMN nexent.knowledge_record_t.preserve_source_file IS 'Whether to preserve uploaded source documents after vectorization'; diff --git a/docker/sql/v2.2.1_0603_add_greeting_fields_to_ag_tenant_agent_t.sql b/docker/sql/v2.2.1_0603_add_greeting_fields_to_ag_tenant_agent_t.sql deleted file mode 100644 index 7786bb902..000000000 --- a/docker/sql/v2.2.1_0603_add_greeting_fields_to_ag_tenant_agent_t.sql +++ /dev/null @@ -1,15 +0,0 @@ --- Migration: Add greeting_message and example_questions columns to ag_tenant_agent_t table --- Date: 2026-06-03 --- Description: Add greeting message and example questions fields for agent chat initial screen - --- Add greeting_message column to ag_tenant_agent_t table -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS greeting_message TEXT; - --- Add example_questions column to ag_tenant_agent_t table -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS example_questions JSONB; - --- Add comments to the columns -COMMENT ON COLUMN nexent.ag_tenant_agent_t.greeting_message IS 'Agent greeting message displayed on chat initial screen'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.example_questions IS 'List of example questions for starting a conversation with this agent'; \ No newline at end of file diff --git a/docker/sql/v2.2.1_0605_add_ag_agent_repository_t.sql b/docker/sql/v2.2.1_0605_add_ag_agent_repository_t.sql deleted file mode 100644 index d719fc5aa..000000000 --- a/docker/sql/v2.2.1_0605_add_ag_agent_repository_t.sql +++ /dev/null @@ -1,96 +0,0 @@ --- Migration: Add ag_agent_repository_t table --- Date: 2026-06-05 --- Description: Agent marketplace repository for frozen shareable agent snapshots. - -SET search_path TO nexent; - -BEGIN; - -CREATE SEQUENCE IF NOT EXISTS nexent.ag_agent_repository_t_agent_repository_id_seq; - -CREATE TABLE IF NOT EXISTS nexent.ag_agent_repository_t ( - agent_repository_id BIGINT NOT NULL DEFAULT nextval('nexent.ag_agent_repository_t_agent_repository_id_seq'), - publisher_tenant_id VARCHAR(100) NOT NULL, - publisher_user_id VARCHAR(100) NOT NULL, - agent_id INTEGER NOT NULL, - source_version_no INTEGER NOT NULL, - name VARCHAR(100) NOT NULL, - display_name VARCHAR(100), - description TEXT, - author VARCHAR(100), - category_id INTEGER, - tags TEXT[], - tool_count INTEGER, - version_label VARCHAR(100), - agent_info_json JSONB NOT NULL, - status VARCHAR(30) DEFAULT 'NOT_SHARED', - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N', - CONSTRAINT ag_agent_repository_t_pkey PRIMARY KEY (agent_repository_id) -); - -ALTER SEQUENCE nexent.ag_agent_repository_t_agent_repository_id_seq - OWNED BY nexent.ag_agent_repository_t.agent_repository_id; - -ALTER TABLE nexent.ag_agent_repository_t OWNER TO root; - -COMMENT ON TABLE nexent.ag_agent_repository_t IS 'Agent marketplace repository for frozen shareable agent snapshots'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.agent_repository_id IS 'Agent repository listing ID, unique primary key'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.publisher_tenant_id IS 'Publisher tenant ID'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.publisher_user_id IS 'Publisher user ID'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.agent_id IS 'Root agent ID from ag_tenant_agent_t; upsert key with publisher_tenant_id'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.source_version_no IS 'Published version number frozen at share time'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.name IS 'Root agent programmatic name for display and search'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.display_name IS 'Root agent display name'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.description IS 'Root agent description'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.author IS 'Agent author'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.category_id IS 'Optional marketplace category ID'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.tags IS 'Marketplace tags'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.tool_count IS 'Total tool count across all agents in the bundle (display only)'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.version_label IS 'Repository entry version label for display (e.g. v1.0)'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.agent_info_json IS 'Frozen ExportAndImportDataFormat snapshot with optional skills'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.status IS 'Listing status: NOT_SHARED (未共享) / PENDING_REVIEW (待审核) / REJECTED (审核驳回) / SHARED (已共享)'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.create_time IS 'Creation time'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.created_by IS 'Creator ID'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.updated_by IS 'Updater ID'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.delete_flag IS 'Soft delete flag: Y/N'; - -CREATE UNIQUE INDEX IF NOT EXISTS uq_agent_repository_tenant_agent_active - ON nexent.ag_agent_repository_t (publisher_tenant_id, agent_id) - WHERE delete_flag = 'N'; - -CREATE INDEX IF NOT EXISTS idx_agent_repository_publisher_delete - ON nexent.ag_agent_repository_t (publisher_tenant_id, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_agent_repository_status_delete - ON nexent.ag_agent_repository_t (status, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_agent_repository_name_delete - ON nexent.ag_agent_repository_t (name, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_agent_repository_tags_gin - ON nexent.ag_agent_repository_t USING GIN (tags); - -CREATE OR REPLACE FUNCTION update_ag_agent_repository_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -COMMENT ON FUNCTION update_ag_agent_repository_update_time() IS 'Auto-update update_time for ag_agent_repository_t'; - -DROP TRIGGER IF EXISTS update_ag_agent_repository_update_time_trigger ON nexent.ag_agent_repository_t; -CREATE TRIGGER update_ag_agent_repository_update_time_trigger -BEFORE UPDATE ON nexent.ag_agent_repository_t -FOR EACH ROW -EXECUTE FUNCTION update_ag_agent_repository_update_time(); - -COMMENT ON TRIGGER update_ag_agent_repository_update_time_trigger ON nexent.ag_agent_repository_t IS 'Trigger to maintain update_time'; - -COMMIT; diff --git a/docker/sql/v2.2.1_0609_add_selected_agent_version_no_to_agent_relation_t.sql b/docker/sql/v2.2.1_0609_add_selected_agent_version_no_to_agent_relation_t.sql deleted file mode 100644 index 9a67c1ab2..000000000 --- a/docker/sql/v2.2.1_0609_add_selected_agent_version_no_to_agent_relation_t.sql +++ /dev/null @@ -1,15 +0,0 @@ --- Migration: Add selected_agent_version_no to ag_agent_relation_t --- Date: 2026-06-09 --- Description: Pin child agent version on parent-child relations at publish time. - -SET search_path TO nexent; - -BEGIN; - -ALTER TABLE nexent.ag_agent_relation_t - ADD COLUMN IF NOT EXISTS selected_agent_version_no INTEGER; - -COMMENT ON COLUMN nexent.ag_agent_relation_t.selected_agent_version_no IS - 'Pinned version of selected_agent_id. NULL = use child current published version at runtime (legacy/draft).'; - -COMMIT; diff --git a/frontend/app/[locale]/agents/components/AgentConfigComp.tsx b/frontend/app/[locale]/agents/components/AgentConfigComp.tsx index 1e750d5eb..13484595f 100644 --- a/frontend/app/[locale]/agents/components/AgentConfigComp.tsx +++ b/frontend/app/[locale]/agents/components/AgentConfigComp.tsx @@ -29,8 +29,6 @@ export default function AgentConfigComp({}: AgentConfigCompProps) { const currentAgentId = useAgentConfigStore((state) => state.currentAgentId); const isCreatingMode = useAgentConfigStore((state) => state.isCreatingMode); const isReadOnly = useAgentConfigStore((state) => state.isReadOnly()); - const selectedTools = useAgentConfigStore((state) => state.editedAgent.tools); - const selectedSkills = useAgentConfigStore((state) => state.editedAgent.skills); const [isMcpModalOpen, setIsMcpModalOpen] = useState(false); const [isSkillModalOpen, setIsSkillModalOpen] = useState(false); @@ -127,12 +125,7 @@ export default function AgentConfigComp({}: AgentConfigCompProps) { - - {t("toolPool.title")} - {selectedTools.length > 0 && ( - - )} - + {t("toolPool.title")} {t("toolPool.tooltip.functionGuide")}} color="#ffffff" @@ -151,14 +144,7 @@ export default function AgentConfigComp({}: AgentConfigCompProps) { - - - {t("skillPool.title")} - {selectedSkills && selectedSkills.length > 0 && ( - - )} - - + {t("skillPool.title")} diff --git a/frontend/app/[locale]/agents/components/agentConfig/McpConfigModal.tsx b/frontend/app/[locale]/agents/components/agentConfig/McpConfigModal.tsx index 41c8baa45..277e85d3d 100644 --- a/frontend/app/[locale]/agents/components/agentConfig/McpConfigModal.tsx +++ b/frontend/app/[locale]/agents/components/agentConfig/McpConfigModal.tsx @@ -80,7 +80,6 @@ export default function McpConfigModal({ const [openApiJson, setOpenApiJson] = useState(""); const [openApiServiceName, setOpenApiServiceName] = useState(""); const [openApiServerUrl, setOpenApiServerUrl] = useState(""); - const [openApiHeadersTemplate, setOpenApiHeadersTemplate] = useState(""); const [importingOpenApi, setImportingOpenApi] = useState(false); const [openapiServices, setOpenapiServices] = useState([]); const [loadingOpenapiServices, setLoadingOpenapiServices] = useState(false); @@ -507,7 +506,6 @@ export default function McpConfigModal({ service_name: openApiServiceName.trim(), server_url: openApiServerUrl.trim(), openapi_json: parsedJson, - headers_template: openApiHeadersTemplate.trim() ? JSON.parse(openApiHeadersTemplate.trim()) : null, }), }); @@ -516,7 +514,6 @@ export default function McpConfigModal({ setOpenApiJson(""); setOpenApiServiceName(""); setOpenApiServerUrl(""); - setOpenApiHeadersTemplate(""); await loadOpenapiServices(); await refreshToolsAndAgents(); } else { @@ -1223,20 +1220,15 @@ export default function McpConfigModal({ style={{ flex: 3 }} /> - setOpenApiHeadersTemplate(e.target.value)} - rows={2} - disabled={actionsLocked || importingOpenApi} - /> - setOpenApiJson(e.target.value)} - rows={6} - disabled={actionsLocked || importingOpenApi} - /> +
+ setOpenApiJson(e.target.value)} + rows={6} + disabled={actionsLocked || importingOpenApi} + /> +
{ - const selectedCount = group.skills.filter(s => originalSelectedSkillIdsSet.has(s.skill_id)).length; - return { key: group.key, label: ( - - - {group.label} - - {selectedCount > 0 && ( - - )} + + {group.label} ), diff --git a/frontend/app/[locale]/agents/components/agentConfig/ToolManagement.tsx b/frontend/app/[locale]/agents/components/agentConfig/ToolManagement.tsx index 62edc3ac8..0cb73de62 100644 --- a/frontend/app/[locale]/agents/components/agentConfig/ToolManagement.tsx +++ b/frontend/app/[locale]/agents/components/agentConfig/ToolManagement.tsx @@ -4,7 +4,7 @@ import { useState, useEffect, useCallback } from "react"; import { useTranslation } from "react-i18next"; import ToolConfigModal from "./tool/ToolConfigModal"; import { ToolGroup, Tool, ToolParam } from "@/types/agentConfig"; -import { Tabs, Collapse, message, Tooltip, Badge } from "antd"; +import { Tabs, Collapse, message, Tooltip } from "antd"; import { useAgentConfigStore } from "@/stores/agentConfigStore"; import { useToolList } from "@/hooks/agent/useToolList"; import { usePrefetchKnowledgeBases } from "@/hooks/useKnowledgeBaseSelector"; @@ -307,29 +307,21 @@ export default function ToolManagement({ // Generate Tabs configuration const tabItems = toolGroups.map((group) => { const label = t(group.label); - const selectedCount = group.subGroups - ? group.subGroups.reduce( - (sum, sg) => sum + sg.tools.filter(t => originalSelectedToolIdsSet.has(t.id)).length, 0) - : group.tools.filter(t => originalSelectedToolIdsSet.has(t.id)).length; return { key: group.key, label: ( - - - {label} - - {selectedCount > 0 && ( - - )} + + {label} ), @@ -359,25 +351,17 @@ export default function ToolManagement({ items={group.subGroups.map((subGroup, index) => ({ key: subGroup.key, label: ( - - - {subGroup.label} - - {subGroup.tools.filter(t => originalSelectedToolIdsSet.has(t.id)).length > 0 && ( - originalSelectedToolIdsSet.has(t.id)).length} - size="small" - color="blue" - /> - )} + + {subGroup.label} ), className: `tool-category-panel ${ diff --git a/frontend/app/[locale]/agents/components/agentConfig/skill/SkillConfigModal.tsx b/frontend/app/[locale]/agents/components/agentConfig/skill/SkillConfigModal.tsx index 9729007e2..6f372e2b4 100644 --- a/frontend/app/[locale]/agents/components/agentConfig/skill/SkillConfigModal.tsx +++ b/frontend/app/[locale]/agents/components/agentConfig/skill/SkillConfigModal.tsx @@ -12,13 +12,13 @@ import { message, Tag, Skeleton, - Tooltip } from "antd"; import { Settings } from "lucide-react"; import { CloseOutlined } from "@ant-design/icons"; import { Skill, SkillParam } from "@/types/agentConfig"; import { KnowledgeBase } from "@/types/knowledgeBase"; +import { Tooltip } from "@/components/ui/tooltip"; import { saveSkillInstance } from "@/services/agentConfigService"; import KnowledgeBaseSelectorModal from "@/components/tool-config/KnowledgeBaseSelectorModal"; import { diff --git a/frontend/app/[locale]/agents/components/agentInfo/AgentGenerateDetail.tsx b/frontend/app/[locale]/agents/components/agentInfo/AgentGenerateDetail.tsx index cd46d2aa3..8b6cd82d7 100644 --- a/frontend/app/[locale]/agents/components/agentInfo/AgentGenerateDetail.tsx +++ b/frontend/app/[locale]/agents/components/agentInfo/AgentGenerateDetail.tsx @@ -1,6 +1,6 @@ "use client"; -import { useState, useEffect, useMemo, useRef } from "react"; +import { useState, useEffect, useMemo, useRef, useCallback } from "react"; import { useTranslation } from "react-i18next"; import { Button, @@ -17,11 +17,9 @@ import { } from "antd"; import { Tabs, TabsList, TabsTrigger, TabsContent } from "@/components/ui/tabs"; import { Zap, Maximize2, Settings2, Sparkles } from "lucide-react"; -import { Textarea } from "@/components/ui/textarea"; import { AgentConfigUpdate, - DEFAULT_AGENT_VERIFICATION_CONFIG, PromptTemplate, } from "@/types/agentConfig"; import { @@ -171,7 +169,6 @@ export default function AgentGenerateDetail({}) { constraintPrompt: editedAgent.constraint_prompt || "", fewShotsPrompt: editedAgent.few_shots_prompt || "", provideRunSummary: editedAgent.provide_run_summary || false, - verificationEnabled: editedAgent.verification_config?.enabled ?? false, businessDescription: editedAgent.business_description || "", businessLogicModelName:editedAgent.business_logic_model_name, businessLogicModelId: editedAgent.business_logic_model_id, @@ -236,7 +233,6 @@ export default function AgentGenerateDetail({}) { setOptimizeModalOpen(true); }; - const renderExpandButton = (type: "duty" | "constraint" | "few-shots") => { return (