diff --git a/backend/src/agent/graph.py b/backend/src/agent/graph.py index cdfdbbc95..3cb80cc28 100644 --- a/backend/src/agent/graph.py +++ b/backend/src/agent/graph.py @@ -153,7 +153,7 @@ def reflection_router(state: OverallState) -> list[Send] | str: ) builder.add_edge("denoising_refiner", END) -# TODO(priority=High, complexity=Medium): [SOTA Deep Research] Graph Wiring +# TODO(priority=High, complexity=Medium, owner=agent): [SOTA Deep Research] Graph Wiring # Add conditional edges to route from 'reflection' or 'update_plan' to 'research_subgraph'. # research_subgraph results should then flow back into 'update_plan' or merge into the state. diff --git a/backend/src/agent/mcp_config.py b/backend/src/agent/mcp_config.py index 8ef9a1e66..1b5b26f7d 100644 --- a/backend/src/agent/mcp_config.py +++ b/backend/src/agent/mcp_config.py @@ -47,26 +47,26 @@ def validate(settings: MCPSettings) -> None: # Fine-grained implementation guide for MCP Integration: # -# TODO(priority=High, complexity=Low): [MCP:1] Define SSE client interface +# TODO(priority=High, complexity=Low, owner=agent): [MCP:1] Define SSE client interface # - Create abstract base class for MCP transport # - Define methods: connect(), disconnect(), send_message(), receive_stream() # -# TODO(priority=High, complexity=Medium): [MCP:2] Implement SSE transport +# TODO(priority=High, complexity=Medium, owner=agent): [MCP:2] Implement SSE transport # - Use httpx or aiohttp for Server-Sent Events # - Handle reconnection with exponential backoff # - Parse SSE event format (event:, data:, id:) # -# TODO(priority=Medium, complexity=Medium): [MCP:3] Connection pooling +# TODO(priority=Medium, complexity=Medium, owner=agent): [MCP:3] Connection pooling # - Maintain pool of persistent connections # - Implement health checks and automatic reconnection # - Thread-safe connection acquisition/release # -# TODO(priority=Medium, complexity=Low): [MCP:4] Error recovery +# TODO(priority=Medium, complexity=Low, owner=agent): [MCP:4] Error recovery # - Catch and log transport errors # - Retry failed tool calls with backoff # - Return graceful fallback on persistent failure # -# TODO(priority=Low, complexity=Low): [MCP:5] Metrics and observability +# TODO(priority=Low, complexity=Low, owner=agent): [MCP:5] Metrics and observability # - Track connection latency, success/failure rates # - Integrate with Langfuse spans class McpConnectionManager: @@ -94,7 +94,7 @@ def get_persistence_tools(self) -> List: ] async def get_tools(self): - # TODO(priority=High, complexity=Medium): [MCP:6] Implement actual SSE tool discovery + # TODO(priority=High, complexity=Medium, owner=agent): [MCP:6] Implement actual SSE tool discovery # - Connect to MCP endpoint from settings # - Fetch tool list via SSE stream # - Convert to LangChain StructuredTool format diff --git a/backend/src/agent/nodes.py b/backend/src/agent/nodes.py index 75bb67896..19c134ab1 100644 --- a/backend/src/agent/nodes.py +++ b/backend/src/agent/nodes.py @@ -1,11 +1,11 @@ -# TODO(priority=Low, complexity=Low): See docs/tasks/upstream_compatibility.md for future splitting of this file into _nodes.py (upstream) and nodes.py (evolved). +# TODO(priority=Low, complexity=Low, owner=agent): See docs/tasks/upstream_compatibility.md for future splitting of this file into _nodes.py (upstream) and nodes.py (evolved). # -# TODO(priority=Medium, complexity=Medium): [SOTA Deep Research] Benchmarking +# TODO(priority=Medium, complexity=Medium, owner=agent): [SOTA Deep Research] Benchmarking # See docs/tasks/04_SOTA_DEEP_RESEARCH_TASKS.md # Subtask: MLE-bench Integration (Evaluate on Kaggle engineering tasks). # Subtask: DeepResearch-Bench Setup (Load tasks from muset-ai space). -# TODO(priority=Medium, complexity=High): Investigate and integrate 'deepagents' patterns if applicable. +# TODO(priority=Medium, complexity=High, owner=agent): Investigate and integrate 'deepagents' patterns if applicable. # See docs/tasks/04_SOTA_DEEP_RESEARCH_TASKS.md # Subtask: Review 'deepagents' repo for relevant nodes (e.g. hierarchical planning). # Subtask: Adapt useful patterns to `backend/src/agent/nodes.py`. @@ -151,7 +151,7 @@ def scoping_node(state: OverallState, config: RunnableConfig) -> OverallState: If yes -> Generates questions and sets status to 'active' (interrupt). If no -> Sets status to 'complete' (proceed). - TODO(priority=High, complexity=High): [SOTA Deep Research] Verify full alignment with Open Deep Research (Clarification Loop). + TODO(priority=High, complexity=High, owner=agent): [SOTA Deep Research] Verify full alignment with Open Deep Research (Clarification Loop). See docs/tasks/04_SOTA_DEEP_RESEARCH_TASKS.md Subtask: Implement `scoping_node` logic: Analyze input query. If ambiguous, generate clarifying questions and interrupt graph. """ @@ -1035,25 +1035,25 @@ def flow_update(state: OverallState, config: RunnableConfig) -> OverallState: Fine-grained implementation guide: - TODO(priority=High, complexity=Low): [flow_update:1] Extract current task from state + TODO(priority=High, complexity=Low, owner=agent): [flow_update:1] Extract current task from state - Read `current_task_idx` and `plan` from state - Get the task object being evaluated - TODO(priority=High, complexity=Medium): [flow_update:2] Analyze task completion + TODO(priority=High, complexity=Medium, owner=agent): [flow_update:2] Analyze task completion - Compare task query against `web_research_result` - Use fuzzy matching or LLM to determine if task is adequately answered - Return completion_score (0.0-1.0) - TODO(priority=High, complexity=Medium): [flow_update:3] Identify knowledge gaps + TODO(priority=High, complexity=Medium, owner=agent): [flow_update:3] Identify knowledge gaps - Parse research results for "unclear", "contradictory", or "insufficient" signals - Generate list of follow-up questions if gaps detected - TODO(priority=Medium, complexity=High): [flow_update:4] DAG expansion logic + TODO(priority=Medium, complexity=High, owner=agent): [flow_update:4] DAG expansion logic - If gaps detected: Create new tasks and insert into plan - If task complete: Mark status='done' and increment current_task_idx - If no more tasks: Set research_complete=True - TODO(priority=Low, complexity=Low): [flow_update:5] Return updated state + TODO(priority=Low, complexity=Low, owner=agent): [flow_update:5] Return updated state - Return dict with updated `plan`, `current_task_idx`, `research_complete` See docs/tasks/04_SOTA_DEEP_RESEARCH_TASKS.md @@ -1157,7 +1157,7 @@ def content_reader(state: OverallState, config: RunnableConfig) -> OverallState: return {"evidence_bank": extracted_evidence} -# TODO(priority=High, complexity=Medium): [SOTA Deep Research] Recursive Trigger +# TODO(priority=High, complexity=Medium, owner=agent): [SOTA Deep Research] Recursive Trigger # Implement logic in reflection or a new 'router' node to decide when to call 'research_subgraph'. # This should happen when a complex sub-topic is identified that requires its own full research loop. def research_subgraph(state: OverallState, config: RunnableConfig) -> OverallState: diff --git a/backend/src/agent/rag.py b/backend/src/agent/rag.py index 1e57f2c4e..67a019b0d 100644 --- a/backend/src/agent/rag.py +++ b/backend/src/agent/rag.py @@ -537,7 +537,7 @@ class Resource: def create_rag_tool(resources): """Legacy compatibility stub - returns None. - TODO(priority=Low, complexity=Medium): [rag:legacy] Replace stub with real implementation + TODO(priority=Low, complexity=Medium, owner=agent): [rag:legacy] Replace stub with real implementation - Migrate callers to use DeepSearchRAG directly - Remove this function once all callers are updated - Update tests that mock this function diff --git a/backend/src/evaluation/deep_research_bench.py b/backend/src/evaluation/deep_research_bench.py index 329cd6044..38658da1d 100644 --- a/backend/src/evaluation/deep_research_bench.py +++ b/backend/src/evaluation/deep_research_bench.py @@ -1,31 +1,31 @@ # Fine-grained implementation guide for DeepResearch-Bench Evaluation: # -# TODO(priority=High, complexity=Low): [deep_bench:1] Dataset loader +# TODO(priority=High, complexity=Low, owner=agent): [deep_bench:1] Dataset loader # - Connect to muset-ai/DeepResearch-Bench on HuggingFace # - Implement load_deep_research_dataset() -> List[Task] # - Each Task: {id, query, gold_report, evaluation_criteria} # -# TODO(priority=High, complexity=Medium): [deep_bench:2] Agent runner +# TODO(priority=High, complexity=Medium, owner=agent): [deep_bench:2] Agent runner # - Import graph from agent.graph # - Configure for full research mode (scoping -> planning -> research -> synthesis) # - Capture final report and all intermediate artifacts # -# TODO(priority=Medium, complexity=High): [deep_bench:3] Report scorer +# TODO(priority=Medium, complexity=High, owner=agent): [deep_bench:3] Report scorer # - Compare generated report against gold_report # - Use metrics: ROUGE-L, BERTScore, factual accuracy (via NLI) # - Return composite score (0.0-1.0) # -# TODO(priority=Medium, complexity=Medium): [deep_bench:4] Citation verifier +# TODO(priority=Medium, complexity=Medium, owner=agent): [deep_bench:4] Citation verifier # - Check that all claims are backed by sources # - Verify source URLs are valid and content matches claims # - Return citation_coverage score # -# TODO(priority=Medium, complexity=Low): [deep_bench:5] Metrics aggregator +# TODO(priority=Medium, complexity=Low, owner=agent): [deep_bench:5] Metrics aggregator # - Aggregate scores across all tasks # - Compute mean, std, percentiles # - Track token usage and latency # -# TODO(priority=Low, complexity=Low): [deep_bench:6] Report generator +# TODO(priority=Low, complexity=Low, owner=agent): [deep_bench:6] Report generator # - Output results to JSON and Markdown # - Generate comparison charts (if multiple runs) # @@ -34,30 +34,30 @@ def evaluate_deep_research(): """Evaluates the agent on DeepResearch-Bench (muset-ai).""" - # TODO(priority=High, complexity=Low): [deep_bench:1] Load dataset + # TODO(priority=High, complexity=Low, owner=agent): [deep_bench:1] Load dataset dataset = [] # load_deep_research_dataset() - # TODO(priority=High, complexity=Medium): [deep_bench:2] Run agent + # TODO(priority=High, complexity=Medium, owner=agent): [deep_bench:2] Run agent results = [] for task in dataset: # report = run_full_research(task.query) # results.append({"task_id": task.id, "report": report}) _ = task # placeholder until implementation is complete - # TODO(priority=Medium, complexity=High): [deep_bench:3] Score reports + # TODO(priority=Medium, complexity=High, owner=agent): [deep_bench:3] Score reports scores = [] # for result in results: # score = score_report(result["report"], gold_report) # scores.append(score) - # TODO(priority=Medium, complexity=Medium): [deep_bench:4] Verify citations + # TODO(priority=Medium, complexity=Medium, owner=agent): [deep_bench:4] Verify citations # for result in results: # citation_score = verify_citations(result["report"]) - # TODO(priority=Medium, complexity=Low): [deep_bench:5] Aggregate + # TODO(priority=Medium, complexity=Low, owner=agent): [deep_bench:5] Aggregate # mean_score = sum(scores) / len(scores) if scores else 0 - # TODO(priority=Low, complexity=Low): [deep_bench:6] Report + # TODO(priority=Low, complexity=Low, owner=agent): [deep_bench:6] Report print("DeepResearch-Bench evaluation not yet implemented") diff --git a/backend/src/evaluation/mle_bench.py b/backend/src/evaluation/mle_bench.py index 22b8805dc..57df18db8 100644 --- a/backend/src/evaluation/mle_bench.py +++ b/backend/src/evaluation/mle_bench.py @@ -1,26 +1,26 @@ # Fine-grained implementation guide for MLE-bench Evaluation: # -# TODO(priority=High, complexity=Low): [mle_bench:1] Dataset loader +# TODO(priority=High, complexity=Low, owner=agent): [mle_bench:1] Dataset loader # - Define path to MLE-bench dataset (HuggingFace or local) # - Implement load_mle_dataset() -> List[Task] # - Each Task: {id, prompt, expected_output, metadata} # -# TODO(priority=High, complexity=Medium): [mle_bench:2] Agent runner +# TODO(priority=High, complexity=Medium, owner=agent): [mle_bench:2] Agent runner # - Import graph from agent.graph # - Run graph.invoke({"messages": [task.prompt]}) # - Capture final output and execution time # -# TODO(priority=Medium, complexity=Medium): [mle_bench:3] Output evaluator +# TODO(priority=Medium, complexity=Medium, owner=agent): [mle_bench:3] Output evaluator # - Compare agent output against expected_output # - Implement exact_match, fuzzy_match, and llm_judge scoring # - Return score (0.0-1.0) per task # -# TODO(priority=Medium, complexity=Low): [mle_bench:4] Metrics aggregator +# TODO(priority=Medium, complexity=Low, owner=agent): [mle_bench:4] Metrics aggregator # - Compute Pass@1 (% tasks with score >= threshold) # - Compute average score across all tasks # - Track latency percentiles (p50, p95, p99) # -# TODO(priority=Low, complexity=Low): [mle_bench:5] Report generator +# TODO(priority=Low, complexity=Low, owner=agent): [mle_bench:5] Report generator # - Output results to JSON and Markdown # - Include per-task breakdown and aggregate stats # @@ -29,27 +29,27 @@ def evaluate_mle_bench(): """Evaluates the agent on MLE-bench tasks.""" - # TODO(priority=High, complexity=Low): [mle_bench:1] Load dataset + # TODO(priority=High, complexity=Low, owner=agent): [mle_bench:1] Load dataset dataset = [] # load_mle_dataset() - # TODO(priority=High, complexity=Medium): [mle_bench:2] Run agent + # TODO(priority=High, complexity=Medium, owner=agent): [mle_bench:2] Run agent results = [] for task in dataset: # output = run_agent(task.prompt) # results.append({"task_id": task.id, "output": output}) _ = task # placeholder until implementation is complete - # TODO(priority=Medium, complexity=Medium): [mle_bench:3] Evaluate + # TODO(priority=Medium, complexity=Medium, owner=agent): [mle_bench:3] Evaluate scores = [] # for result in results: # score = evaluate_output(result["output"], ...) # scores.append(score) - # TODO(priority=Medium, complexity=Low): [mle_bench:4] Aggregate + # TODO(priority=Medium, complexity=Low, owner=agent): [mle_bench:4] Aggregate # pass_at_1 = sum(1 for s in scores if s >= 0.5) / len(scores) # avg_score = sum(scores) / len(scores) - # TODO(priority=Low, complexity=Low): [mle_bench:5] Report + # TODO(priority=Low, complexity=Low, owner=agent): [mle_bench:5] Report print("MLE-bench evaluation not yet implemented") diff --git a/backend/tests/test_mcp.py b/backend/tests/test_mcp.py index 150580cec..f9ea76c1e 100644 --- a/backend/tests/test_mcp.py +++ b/backend/tests/test_mcp.py @@ -4,20 +4,20 @@ # Fine-grained implementation guide for MCP Tests: # -# TODO(priority=Medium, complexity=Low): [test_mcp:1] Test disabled MCP returns empty list +# TODO(priority=Medium, complexity=Low, owner=agent): [test_mcp:1] Test disabled MCP returns empty list # - Create MCPSettings with enabled=False # - Verify get_tools_from_mcp returns [] # -# TODO(priority=Medium, complexity=Medium): [test_mcp:2] Test connection error handling +# TODO(priority=Medium, complexity=Medium, owner=agent): [test_mcp:2] Test connection error handling # - Mock SSEConnection to raise ConnectionError # - Verify graceful fallback (empty list, logged warning) # -# TODO(priority=Medium, complexity=Medium): [test_mcp:3] Test tool whitelist filtering +# TODO(priority=Medium, complexity=Medium, owner=agent): [test_mcp:3] Test tool whitelist filtering # - Load multiple tools from mock MCP # - Set tool_whitelist to subset # - Verify only whitelisted tools returned # -# TODO(priority=Low, complexity=Medium): [test_mcp:4] Test tool execution with real MCP server +# TODO(priority=Low, complexity=Medium, owner=agent): [test_mcp:4] Test tool execution with real MCP server # - Skip if MCP_ENDPOINT not set (integration test) # - Connect to real server, call a tool, verify response format # diff --git a/docs/benchmarks/PLAN.md b/docs/benchmarks/PLAN.md index 756305a7d..f337a05d2 100644 --- a/docs/benchmarks/PLAN.md +++ b/docs/benchmarks/PLAN.md @@ -1,4 +1,4 @@ -# TODO(priority=High, complexity=Large): Benchmarking & Evaluation Framework +# TODO(priority=High, complexity=Large, owner=agent): Benchmarking & Evaluation Framework We need to implement a systematic evaluation framework to measure improvements in report quality, relevance, and accuracy. diff --git a/scripts/extract_todos_structured.py b/scripts/extract_todos_structured.py index f6daf1313..500b8db63 100644 --- a/scripts/extract_todos_structured.py +++ b/scripts/extract_todos_structured.py @@ -12,7 +12,7 @@ def extract_todos(root_dir): dirs[:] = [d for d in dirs if d not in exclude_dirs] for file in files: - if file.endswith(('.py', '.tsx', '.ts', '.js', '.jsx', '.md')): + if file.endswith(('.py', '.tsx', '.ts', '.js', '.jsx', '.md')) and file != 'extract_todos_structured.py': filepath = os.path.join(root, file) try: with open(filepath, 'r', encoding='utf-8', errors='ignore') as f: @@ -22,7 +22,7 @@ def extract_todos(root_dir): # Simple parser content = line.strip() # Try to parse structured TODOs if they exist - # Format: TODO(priority=, complexity=): + # Format: TODO(priority=, complexity=, owner=): priority = "Unknown" complexity = "Unknown" diff --git a/scripts/find_stale_unused_deps.py b/scripts/find_stale_unused_deps.py new file mode 100755 index 000000000..470c20dbc --- /dev/null +++ b/scripts/find_stale_unused_deps.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +import json +import subprocess +import os +import re +from datetime import datetime, timedelta, timezone + +# --- CONFIG --- +AGE_THRESHOLD_DAYS = 90 +REPORT_FILE = "unused_deps_report.txt" + +# --- HELPERS --- +def get_file_blame_lines(filepath): + """Returns a list of dicts with line number, commit date, and content.""" + try: + output = subprocess.check_output( + ["git", "blame", "--line-porcelain", filepath], + universal_newlines=True, stderr=subprocess.DEVNULL + ) + lines = [] + current_line = {} + for line in output.splitlines(): + if line.startswith("author-time "): + current_line['time'] = int(line.split(" ")[1]) + elif line.startswith("\t"): + current_line['content'] = line[1:] + lines.append(current_line) + current_line = {} + return lines + except Exception as e: + print(f"Error blaming {filepath}: {e}") + return [] + +def get_age_days(timestamp): + commit_date = datetime.fromtimestamp(timestamp, tz=timezone.utc) + now = datetime.now(tz=timezone.utc) + return (now - commit_date).days + +def is_used(dep_name, search_dirs, extensions): + """Simple grep-based heuristic to check if a dependency is imported.""" + # This is a naive MVP check. + # Convert hyphens to underscores for python (e.g. langchain-core -> langchain_core) + py_dep = dep_name.replace('-', '_') + + # We will search for occurrences of dep_name or py_dep in source files. + for d in search_dirs: + for root, dirs, files in os.walk(d): + if 'node_modules' in dirs: + dirs.remove('node_modules') + if '.venv' in dirs: + dirs.remove('.venv') + for file in files: + if any(file.endswith(ext) for ext in extensions): + filepath = os.path.join(root, file) + try: + with open(filepath, 'r', encoding='utf-8') as f: + content = f.read() + if dep_name in content or py_dep in content: + return True + except Exception: + pass + return False + +# --- FRONTEND (package.json) --- +def check_frontend(): + print("Checking frontend...") + pkg_json = "frontend/package.json" + if not os.path.exists(pkg_json): + return [] + + with open(pkg_json, 'r', encoding='utf-8') as f: + pkg = json.load(f) + + deps = list(pkg.get("dependencies", {}).keys()) + list(pkg.get("devDependencies", {}).keys()) + + blame = get_file_blame_lines(pkg_json) + + results = [] + for dep in deps: + # Exclude internal or highly common toolings if desired, but for MVP check all + if dep in ['react', 'react-dom', 'typescript', 'vite']: continue + + # Find oldest insertion of this dep in package.json + age = 0 + for b_line in blame: + if f'"{dep}"' in b_line['content']: + age = max(age, get_age_days(b_line.get('time', 0))) + + if age > AGE_THRESHOLD_DAYS: + # Check if used + if not is_used(dep, ['frontend/src'], ['.ts', '.tsx', '.js', '.jsx', '.css']): + results.append(f"[Frontend] {dep} (Age: {age} days)") + return results + +# --- BACKEND (pyproject.toml / requirements.txt) --- +def check_backend(): + print("Checking backend...") + pyproj = "backend/pyproject.toml" + results = [] + + if os.path.exists(pyproj): + with open(pyproj, 'r', encoding='utf-8') as f: + content = f.read() + + deps = [] + in_deps = False + for line in content.splitlines(): + line = line.strip() + if line == 'dependencies = [': + in_deps = True + continue + if in_deps and line == ']': + in_deps = False + continue + if in_deps and line.startswith('"'): + # parse dependency name + dep = line.split('"')[1] + # clean versions like "fastapi>=0.100" -> "fastapi" + dep = re.split(r'[=><~]', dep)[0] + deps.append(dep) + + blame = get_file_blame_lines(pyproj) + + for dep in deps: + # Skip python runtime dep or core frameworks + if dep in ['python', 'pytest']: continue + age = 0 + for b_line in blame: + if dep in b_line['content']: + age = max(age, get_age_days(b_line.get('time', 0))) + + if age > AGE_THRESHOLD_DAYS: + if not is_used(dep, ['backend/src', 'backend/tests'], ['.py']): + results.append(f"[Backend] {dep} (Age: {age} days)") + return results + +# --- RUN --- +def main(): + report = [] + report.extend(check_frontend()) + report.extend(check_backend()) + + with open(REPORT_FILE, 'w') as f: + f.write("Unused Dependencies (>90 days old):\n") + f.write("======================================\n") + if not report: + f.write("None found.\n") + for item in report: + f.write(f"- {item}\n") + + print(f"Done. Report saved to {REPORT_FILE}") + +if __name__ == "__main__": + main()