From 0630a156c7311bf1bf7df60c5fc8c96befbfeb56 Mon Sep 17 00:00:00 2001 From: Peter Bircak Date: Wed, 28 Oct 2026 10:22:48 +0000 Subject: [PATCH] fix: fall back to SGAI_API_KEY env var in get_api_key Local stdio runs (e.g. Claude Code via .mcp.json) had no way to pass the key: get_api_key only checked the X-API-Key header and Smithery session config, despite README/ConfigSchema documenting SGAI_API_KEY. Every tool call failed with "API key is required". Also adds a local-setup SOP under .agent/sop. Co-Authored-By: Claude Opus 5.5 (1M context) --- .agent/README.md | 10 +++++- .agent/sop/local_mcp_setup.md | 61 +++++++++++++++++++++++++++++++++++ src/scrapegraph_mcp/server.py | 16 ++++++--- 3 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 .agent/sop/local_mcp_setup.md diff --git a/.agent/README.md b/.agent/README.md index fbb8a16..9ada696 100644 --- a/.agent/README.md +++ b/.agent/README.md @@ -34,7 +34,11 @@ Complete Model Context Protocol integration documentation: ### SOP Documentation (`sop/`) -*Future: Standard operating procedures (e.g., adding new tools, testing)* +#### [Local MCP Setup](./sop/local_mcp_setup.md) +Run the server locally over stdio and register it with Claude Code for this project (`.mcp.json`): +- **uv + Python 3.12** - `uv sync --python 3.12` from `uv.lock` +- **Smoke test** - JSON-RPC `tools/list` over stdio +- **`.mcp.json`** - `scrapegraph-local` server, key pulled in via `${SGAI_API_KEY}` --- @@ -378,6 +382,10 @@ npx @modelcontextprotocol/inspector scrapegraph-mcp ## 📅 Changelog +### October 2026 +- ✅ `get_api_key()` now falls back to the `SGAI_API_KEY` env var for local stdio runs. Before this, only the header or Smithery config worked, even though the docs said otherwise. +- ✅ Added [Local MCP Setup SOP](./sop/local_mcp_setup.md) (uv + `.mcp.json` for Claude Code) + ### April 2026 - ✅ Migrated MCP client and tools to **API v2** ([scrapegraph-py#84](https://github.com/ScrapeGraphAI/scrapegraph-py/pull/84)): base `https://v2-api.scrapegraphai.com/api`, `SGAI-APIKEY` header (matches SDK wire format), new crawl/monitor/credits/history tools; removed sitemap, agentic_scrapper, status polling tools. Env vars aligned with SDK: `SGAI_API_URL`, `SGAI_TIMEOUT` (legacy alias `SGAI_TIMEOUT_S` still honored). - ✅ Added `monitor_activity` tool for paginated tick history (GET /monitor/:id/activity), mirroring `sgai.monitor.activity()` in scrapegraph-py v2. diff --git a/.agent/sop/local_mcp_setup.md b/.agent/sop/local_mcp_setup.md new file mode 100644 index 0000000..8ec5438 --- /dev/null +++ b/.agent/sop/local_mcp_setup.md @@ -0,0 +1,61 @@ +# SOP: Run the MCP Server Locally in Claude Code + +How to run this repo's server over stdio and register it with Claude Code for this project only. + +## Prerequisites + +- Python 3.10+ (`requires-python` in `pyproject.toml`). macOS system Python is 3.9, so use uv to get a managed interpreter. +- A ScrapeGraph API key from [dashboard.scrapegraphai.com](https://dashboard.scrapegraphai.com) + +## Steps + +1. **Install uv** (installs to `~/.local/bin`, no sudo needed): + ```bash + curl -LsSf https://astral.sh/uv/install.sh | sh + ``` + +2. **Create the environment** from `uv.lock` (uv downloads CPython 3.12 if it's missing): + ```bash + uv sync --python 3.12 + ``` + +3. **Smoke test over stdio.** The last line of output should be a `tools/list` result with 17 tools: + ```bash + (printf '%s\n' \ + '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"smoke","version":"0"}}}' \ + '{"jsonrpc":"2.0","method":"notifications/initialized"}' \ + '{"jsonrpc":"2.0","id":2,"method":"tools/list"}'; sleep 5) \ + | SGAI_API_KEY=dummy uv run scrapegraph-mcp 2>/dev/null | tail -1 + ``` + +4. **Register with Claude Code** by adding `.mcp.json` at the repo root: + ```json + { + "mcpServers": { + "scrapegraph-local": { + "command": "uv", + "args": ["run", "scrapegraph-mcp"], + "env": { "SGAI_API_KEY": "${SGAI_API_KEY}" } + } + } + } + ``` + Claude Code fills in `${SGAI_API_KEY}` from your shell when it launches the server, so the key never goes into the file. `uv run` resolves the project from the working directory, which Claude Code sets to the repo root. + +5. **Provide the key and approve the server:** + ```bash + echo 'export SGAI_API_KEY=your-api-key' >> ~/.zshrc && source ~/.zshrc + claude # approve "scrapegraph-local" when prompted + ``` + +## Verify + +- `claude mcp list` shows `scrapegraph-local: uv run scrapegraph-mcp - ✔ Connected` +- Calling the `credits` tool in a session returns your balance instead of a 401. + +## Troubleshooting + +- **`Missing environment variables: SGAI_API_KEY`**: the key isn't exported in the shell that started `claude`. +- **`ScapeGraph API key is required`**: `get_api_key()` in `server.py` checks, in order, the `X-API-Key` header, the Smithery `scrapegraphApiKey` session config, and then `SGAI_API_KEY`. If you see this error, none of the three is set in the server's environment. +- **`⏸ Pending approval`**: start `claude` in the repo and approve the project server (or use `/mcp`). +- **Server fails to start from another cwd**: change the args to `["run", "--directory", "/abs/path/to/repo", "scrapegraph-mcp"]`. diff --git a/src/scrapegraph_mcp/server.py b/src/scrapegraph_mcp/server.py index ec8249d..bd3cf21 100644 --- a/src/scrapegraph_mcp/server.py +++ b/src/scrapegraph_mcp/server.py @@ -572,11 +572,12 @@ class ConfigSchema(BaseModel): def get_api_key(ctx: Context) -> str: """ - Get the API key from HTTP header or MCP session config. + Get the API key from HTTP header, MCP session config, or environment. - Supports two modes: + Supports three modes: - HTTP mode (Render): API key from 'X-API-Key' header via mcp-remote - Stdio mode (Smithery): API key from session_config.scrapegraph_api_key + - Stdio mode (local): API key from the SGAI_API_KEY environment variable Args: ctx: FastMCP context @@ -607,11 +608,18 @@ def get_api_key(ctx: Context) -> str: logger.info("API key retrieved from session config") return str(api_key) - logger.error("No API key found in header or session config") + # Fall back to environment (for plain local stdio runs) + api_key = os.environ.get("SGAI_API_KEY") + if api_key: + logger.info("API key retrieved from SGAI_API_KEY environment variable") + return api_key + + logger.error("No API key found in header, session config, or environment") raise ValueError( "ScapeGraph API key is required. Please provide it via:\n" "- HTTP header 'X-API-Key' (for remote server via mcp-remote)\n" - "- MCP config 'scrapegraphApiKey' (for Smithery/local stdio)" + "- MCP config 'scrapegraphApiKey' (for Smithery/local stdio)\n" + "- SGAI_API_KEY environment variable (for local stdio)" )