From ac4d364f9c4f913e90797d1c27872cb53c513095 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 20 Sep 2026 10:34:54 +0000 Subject: [PATCH] Write MCP stdio as NDJSON, not Content-Length Cursor live tool discovery expects newline-delimited JSON. Keep a dual-mode reader. Opt-in overlay MCP uses a workspace-relative launcher path for local Desktop. Co-authored-by: ale93.moro --- README.md | 6 +-- docs/ARCHITECTURE.md | 2 +- overlay/mcp.optional.json | 2 +- tests/test_tooling.py | 9 +++-- tools/docs_mcp_server.py | 77 +++++++++++++++------------------------ 5 files changed, 40 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index c40b81e..d90f009 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ make demo # catch the bad scheduler, pass the good one, MCP, tests make demo-contribute # first-contribution journey (KEEP=1 leaves the new files) make check # run the gate on the whole repo (exit code = # blocking) make test # contract tests — zero third-party installs needed -make mcp # self-test the diffusers-docs MCP server (Content-Length handshake) +make mcp # self-test the diffusers-docs MCP server (NDJSON handshake) make demo-maintain # add a rule, rebuild, watch it propagate to every surface (req #4) make grokbot ROLE=qa # role briefing from sample gate JSON (does not gate) make grokbot-pack # paste-ready Grok Bot iPhone/desktop profiles @@ -128,7 +128,7 @@ The 90-second demo (`make demo`) shows the whole arc: 2. The scaffolded, convention-correct version (`examples/scaffolded_scheduler/`) — **0 findings**. 3. The `diffusers-docs` MCP server answers a grounded query over the library's - docs (Content-Length JSON-RPC, the stdio framing Cursor uses). + docs (newline-delimited JSON-RPC, the stdio framing Cursor uses). 4. Contract tests — green, with numeric determinism skipped cleanly when torch isn't installed. @@ -190,7 +190,7 @@ Each upstream rule cites what it was checked against in its `source:` field. conventions/rules.yaml the single source of truth (component-tagged) tools/convention_check.py the runnable gate (AST + regex) tools/build_projections.py renders every audience surface (per-component .mdc) -tools/docs_mcp_server.py the diffusers-docs MCP server (Content-Length JSON-RPC) +tools/docs_mcp_server.py the diffusers-docs MCP server (NDJSON JSON-RPC) tools/demo_contribute.py CLI twin of the live `/scaffold` contribution knowledge/diffusers-docs/ seed doc corpus for the MCP (override with a real checkout) templates/ scaffold templates + "add a component" guide diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0601484..141bb06 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -42,7 +42,7 @@ fork source (`tools/verify_scheduler_contract.py`). | **AST for structural rules** | Class bases, method presence, `@register_to_config`, mutable defaults, docstrings are *structure*. AST doesn't false-positive on a comment or a reformat the way grep does. Regex is used only for line-level lexical patterns. | | **`.cursorignore` for boundaries** | Requirement 3 says "don't pull in context from outside approved boundaries." `.cursorignore` is exactly that mechanism — it removes paths from the agent's context window, so a scaffold can't copy the known-bad fixture (or, in a customer repo, vendored/secret/generated code) toward prod. | | **`.cursor/hooks.json` `afterFileEdit`** | Runs the gate on the edited file the moment the agent touches it — the same script CI runs — so feedback is instant and CI surprises are rare. The hook reads Cursor's stdin JSON (`file_path`, `edits`). | -| **MCP Content-Length stdio (opt-in on the fork)** | Cursor's local MCP transport is JSON-RPC with `Content-Length` framing, not ad-hoc NDJSON. `tools/docs_mcp_server.py` speaks that framing. Cloud stdio cannot set `cwd` or expand `${workspaceFolder}`, and Hub HTTP MCP is Hub search + OAuth — so the **fork overlay defaults to empty `mcpServers`**. Kit Desktop still wires stdio via `.cursor/mcp-diffusers-docs.py`. Live grounding is Read/Grep on `scheduling_euler_discrete.py` / `scheduling_ddpm.py`, then the gate. | +| **MCP NDJSON stdio (opt-in on the fork)** | Cursor's local MCP transport is JSON-RPC as newline-delimited JSON. `tools/docs_mcp_server.py` writes NDJSON (it still *reads* Content-Length). Cloud stdio cannot set `cwd` or expand `${workspaceFolder}`, and Hub HTTP MCP is Hub search + OAuth — so the **fork overlay defaults to empty `mcpServers`**. Kit Desktop still wires stdio via `.cursor/mcp-diffusers-docs.py`. Live grounding is Read/Grep on `scheduling_euler_discrete.py` / `scheduling_ddpm.py`, then the gate. | | **One script, `--json` + exit code** | Exit code = number of blocking findings makes it a drop-in CI gate anywhere; `--json` feeds dashboards / PR annotations. No GPU, no downloads, no network → cheapest runner. | | **Tests in stdlib `unittest`** | Runs on a fresh clone with zero installs (structural contract), and layers in numeric determinism when torch is present. The demo can't fail because a wheel didn't download. | diff --git a/overlay/mcp.optional.json b/overlay/mcp.optional.json index ba79ce1..9080a64 100644 --- a/overlay/mcp.optional.json +++ b/overlay/mcp.optional.json @@ -3,7 +3,7 @@ "diffusers-docs": { "type": "stdio", "command": "python3", - "args": ["-u", "/workspace/.cursor/mcp-diffusers-docs.py"], + "args": ["-u", ".cursor/mcp-diffusers-docs.py"], "env": { "PYTHONUNBUFFERED": "1" } diff --git a/tests/test_tooling.py b/tests/test_tooling.py index d06719c..94a2463 100644 --- a/tests/test_tooling.py +++ b/tests/test_tooling.py @@ -55,7 +55,7 @@ def test_env_docs_root_wins(self): class TestMcpFraming(unittest.TestCase): - def test_selftest_uses_content_length(self): + def test_selftest_emits_ndjson(self): proc = subprocess.run( [sys.executable, str(ROOT / "tools" / "docs_mcp_server.py"), "--selftest"], cwd=ROOT, @@ -64,7 +64,9 @@ def test_selftest_uses_content_length(self): check=False, ) self.assertEqual(proc.returncode, 0, proc.stderr + proc.stdout) - self.assertIn("Content-Length", proc.stdout) + self.assertNotIn("Content-Length", proc.stdout) + self.assertIn("NDJSON", proc.stdout) + self.assertIn("search_docs", proc.stdout) class TestOverlayMcpDefaults(unittest.TestCase): @@ -82,7 +84,7 @@ def test_overlay_mcp_optional_lists_opt_in_servers(self): stdio = servers["diffusers-docs"] self.assertEqual(stdio.get("type"), "stdio") self.assertEqual(stdio["command"], "python3") - self.assertEqual(stdio["args"], ["-u", "/workspace/.cursor/mcp-diffusers-docs.py"]) + self.assertEqual(stdio["args"], ["-u", ".cursor/mcp-diffusers-docs.py"]) self.assertEqual(servers["huggingface"].get("url"), "https://huggingface.co/mcp") def test_attach_copies_empty_default_and_optional(self): @@ -130,6 +132,7 @@ def test_launcher_serves_from_unrelated_cwd(self): ) self.assertEqual(proc.returncode, 0, proc.stderr.decode("utf-8", "replace")) self.assertIn(b"search_docs", proc.stdout) + self.assertNotIn(b"Content-Length", proc.stdout) self.assertNotIn(b"${workspaceFolder}", proc.stdout) diff --git a/tools/docs_mcp_server.py b/tools/docs_mcp_server.py index 2aff24c..a65bb93 100644 --- a/tools/docs_mcp_server.py +++ b/tools/docs_mcp_server.py @@ -9,13 +9,14 @@ by pointing it at more docs, never by adding new machinery. It complements the convention gate: the gate stops WRONG code; doc-search improves DISCOVERY. -TRANSPORT: MCP stdio — JSON-RPC 2.0 with Content-Length framing (the LSP-style -stdio Cursor uses), plus an NDJSON fallback for ad-hoc CLI testing. Cursor -Desktop launches it from .cursor/mcp.json via .cursor/mcp-diffusers-docs.py -(no ${workspaceFolder} — Cloud stdio does not expand it and cannot set cwd). -Implemented with the standard library only, so it runs on a fresh clone / -GPU-less CI with zero installs (a hard PyPI-free constraint we actually hit -while building this). +TRANSPORT: MCP stdio — JSON-RPC 2.0 as newline-delimited JSON (what Cursor +expects on stdio). The reader still accepts Content-Length (LSP-style) as +well as NDJSON, so older clients keep working. Cursor Desktop launches it +from .cursor/mcp.json via .cursor/mcp-diffusers-docs.py (workspace-relative +arg; no ${workspaceFolder} — Cloud stdio does not expand it and cannot set +cwd). Implemented with the standard library only, so it runs on a fresh +clone / GPU-less CI with zero installs (a hard PyPI-free constraint we +actually hit while building this). Methods implemented: initialize, notifications/initialized, ping, tools/list, tools/call (tool: search_docs). @@ -183,13 +184,10 @@ def _handle(msg: dict): return None -def _write_message(stdout, msg: dict) -> None: - """Write one JSON-RPC message with Content-Length framing (byte length).""" - body = json.dumps(msg, ensure_ascii=False).encode("utf-8") - header = f"Content-Length: {len(body)}\r\n\r\n".encode("ascii") - if hasattr(stdout, "buffer"): - stdout = stdout.buffer - stdout.write(header + body) +def _write_message(msg, stdout=None): + """Write one JSON-RPC message as newline-delimited JSON (MCP stdio spec).""" + stdout = stdout if stdout is not None else sys.stdout + stdout.write(json.dumps(msg, separators=(",", ":")) + "\n") stdout.flush() @@ -243,10 +241,10 @@ def _read_message(stdin): def serve(stdin=None, stdout=None): - """Run the MCP stdio loop (Content-Length framing, NDJSON fallback).""" + """Run the MCP stdio loop (NDJSON write; Content-Length or NDJSON read).""" stdin = stdin if stdin is not None else sys.stdin stdout = stdout if stdout is not None else sys.stdout - # Anything printed to stdout breaks Content-Length framing. Keep logs on stderr. + # Anything extra on stdout breaks NDJSON. Keep logs on stderr. if stdout is sys.stdout: sys.stdout.flush() while True: @@ -258,7 +256,7 @@ def serve(stdin=None, stdout=None): break resp = _handle(msg) if resp is not None: - _write_message(stdout, resp) + _write_message(resp, stdout) def _frame(msg: dict) -> bytes: @@ -266,38 +264,18 @@ def _frame(msg: dict) -> bytes: return f"Content-Length: {len(body)}\r\n\r\n".encode("ascii") + body -def _parse_framed(blob: bytes) -> list[dict]: - """Parse Content-Length framed responses from a bytes blob.""" - out = [] - i = 0 - lower = blob.lower() - needle = b"content-length:" - while True: - idx = lower.find(needle, i) - if idx < 0: - break - header_end = blob.find(b"\r\n\r\n", idx) - sep_len = 4 - if header_end < 0: - header_end = blob.find(b"\n\n", idx) - sep_len = 2 - if header_end < 0: - break - header = blob[idx:header_end].decode("ascii", errors="replace") - try: - length = int(header.split(":", 1)[1].strip().split()[0]) - except (ValueError, IndexError): - break - start = header_end + sep_len - body = blob[start:start + length] - out.append(json.loads(body.decode("utf-8"))) - i = start + length - return out +def _parse_ndjson(blob) -> list[dict]: + """Parse newline-delimited JSON-RPC responses.""" + if isinstance(blob, bytes): + text = blob.decode("utf-8") + else: + text = blob + return [json.loads(ln) for ln in text.splitlines() if ln.strip()] # --------------------------------------------------------------------------- # def _selftest() -> int: - """Simulate the Cursor handshake end-to-end with Content-Length framing.""" + """Simulate the Cursor handshake end-to-end; parse NDJSON responses.""" import io reqs = [ {"jsonrpc": "2.0", "id": 1, "method": "initialize", @@ -312,10 +290,13 @@ def _selftest() -> int: "params": {"name": "search_docs", "arguments": json.dumps({"query": "SchedulerMixin", "k": 1})}}, ] + # Reader still accepts Content-Length; writer emits NDJSON. stdin = io.BytesIO(b"".join(_frame(r) for r in reqs)) - stdout = io.BytesIO() + stdout = io.StringIO() serve(stdin, stdout) - messages = _parse_framed(stdout.getvalue()) + raw = stdout.getvalue() + assert "Content-Length" not in raw, raw + messages = _parse_ndjson(raw) assert messages[0]["result"]["serverInfo"]["name"] == "diffusers-docs", "initialize failed" assert messages[0]["result"]["protocolVersion"] == "2025-03-26", "protocol negotiate failed" assert messages[1]["result"]["tools"][0]["name"] == "search_docs", "tools/list failed" @@ -323,7 +304,7 @@ def _selftest() -> int: assert messages[3]["result"]["prompts"] == [], "prompts/list should be empty, not an error" assert "content" in messages[4]["result"], "tools/call failed" assert "content" in messages[5]["result"], "tools/call JSON-string arguments failed" - print("MCP self-test OK: initialize -> list -> search_docs (Content-Length).") + print("MCP self-test OK: initialize -> list -> search_docs (NDJSON).") print(" tools/call returned:\n ", messages[4]["result"]["content"][0]["text"].replace("\n", "\n ")[:400]) return 0