feat: document reader: /api/v1/read, hosted MCP read_url, readm3 read (0.32.0)

README.md

@@ -358,6 +358,31 @@ same scan (`level` is `"secret"`, `"personal"` or `null`, plus `findings`); the
 reports and never refuses, so a script decides for itself. A clean scan is not a
 guarantee: the check knows common shapes, not every secret.
 
+## Read a document at a URL
+
+`readm3 read URL` prints the text of a PDF, a web page, a Word, Excel, PowerPoint,
+OpenDocument or EPUB file, or any text file, so an agent or a script can summarize a link
+without a PDF library of its own:
+
+    readm3 read https://arxiv.org/pdf/2106.01820
+    curl -s "https://readm3.com/api/v1/read?url=https://arxiv.org/pdf/2106.01820"
+
+The API is `GET /api/v1/read?url=` or `POST /api/v1/read {url, maxChars?}` and answers
+`title`, `kind` (pdf, html, office, text), `pages`, `bytes`, `chars`, `truncated` and
+`text` (100,000 characters by default, up to 400,000). Agents get it as the `read_url`
+tool, both in `readm3 mcp` and on the hosted MCP server at **https://readm3.com/mcp**
+(Streamable HTTP, no login, no session).
+
+No account is needed. Each address gets 20 reads an hour free; after that the answer is
+`402 Payment Required` with an x402 offer for a $1 reader pass for the day (USDC through
+CoinPay, sales page at `/read/pass`), which a pass or a Profullstack Power Key in
+`Authorization: Bearer` or `x-crawl-pass` then skips. A signed-in account (session or API
+token) is not metered. Only public addresses are fetched: private, loopback and
+link-local hosts are refused on the first URL and on every redirect, and a document may
+be up to 25 MB. A scanned PDF with no text layer is refused rather than guessed at.
+Operators set `READM3_FREE_READS`, `READM3_FREE_READS_WINDOW` (seconds) and
+`READM3_READ_PRICE_CENTS`; payments need `COINPAY_X402_KEY` and `CRAWL_PAY_TO`.
+
 ## Editor
 
 Open https://readm3.com/viewer for a plain, fast text editor in your browser, with the

desktop/package.json

@@ -1,7 +1,7 @@
 {
   "name": "readm3-desktop",
   "productName": "readm3",
-  "version": "0.31.0",
+  "version": "0.32.0",
   "private": true,
   "description": "readm3 as a desktop app: a plain, fast text editor for Markdown and code.",
   "author": "Profullstack, Inc. <hello@profullstack.com>",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/readm3",
-  "version": "0.31.0",
+  "version": "0.32.0",
   "description": "Markdown editing and sharing for people and teams. Web/PWA, CLI, API and MCP with version history, view/edit links, and a terminal reader.",
   "type": "module",
   "license": "MIT",
@@ -51,7 +51,8 @@
     "@profullstack/synconfig": "^0.1.3",
     "@profullstack/x402-gateway": "^0.16.1",
     "highlight.js": "^11.12.0",
-    "marked": "^18.1.1"
+    "marked": "^18.1.1",
+    "unpdf": "2.1.0"
   },
   "devDependencies": {
     "@anthropic-ai/sdk": "^1.3.0",

server/api.ts

@@ -7,6 +7,7 @@ import { createPaste, deletePaste, readPaste } from "./pastes.ts";
 import { binaryType } from "../src/code.ts";
 import type { Billing } from "./billing.ts";
 import { PACKS_USD, type Payments } from "./payments.ts";
+import { readUrl } from "./read.ts";
 
 /** Paid AI summaries and the CoinPay top-ups that fund them. */
 export type Money = { billing: Billing; payments: Payments };
@@ -163,6 +164,19 @@ export function createApi(
       if (user) store.ensureWorkspace(user);
       const path = url.pathname.slice(8);
       if (path === "me" && request.method === "GET") return json({ user });
+      // The document reader: GET ?url= or POST {url, maxChars}. No account needed;
+      // server/read-gate.ts meters callers before they get here.
+      if (path === "read") {
+        if (request.method !== "GET" && request.method !== "POST") throw new HttpError(405, "Method not allowed.");
+        limit(`read:${user?.id ?? ip}`, user ? 60 : 20);
+        const args: Record<string, unknown> = request.method === "GET"
+          ? Object.fromEntries(url.searchParams)
+          : await body(request, 16 * 1024);
+        if (typeof args.url !== "string" || !args.url) throw new HttpError(400, "url is required.");
+        const maxChars = args.maxChars === undefined ? undefined : Number(args.maxChars);
+        if (maxChars !== undefined && !Number.isFinite(maxChars)) throw new HttpError(400, "maxChars must be a number.");
+        return json(await readUrl(args.url, { maxChars }));
+      }
       // The shared @profullstack/profile fields; private to the signed-in account.
       if (path === "me/profile") {
         const method = request.method as "GET" | "PUT" | "PATCH";

server/read-gate.ts

@@ -0,0 +1,72 @@
+/**
+ * The meter on the document reader (`/api/v1/read` and the hosted MCP at `/mcp`).
+ *
+ * No account is needed: every caller gets READM3_FREE_READS reads per
+ * READM3_FREE_READS_WINDOW seconds (default 20 an hour, by address). Past that
+ * the answer is 402 with an x402 offer for a reader pass (READM3_READ_PRICE_CENTS,
+ * default $1 for a day), payable in USDC through CoinPay, rather than a 429. A pass
+ * or a Profullstack Power Key reads unmetered, and so does a signed-in readm3
+ * account (session cookie or API token): accounts are throttled per minute by the
+ * API instead. Payments switch on only with COINPAY_X402_KEY and CRAWL_PAY_TO;
+ * without them the free allowance still applies and running out is still a 402.
+ */
+import { createGateway } from "@profullstack/x402-gateway";
+
+export const READ_PATHS = ["/api/v1/read", "/mcp"];
+
+export type ReadGate = { handle: (request: Request) => Promise<Response | null> };
+
+const number = (value: string | undefined, fallback: number) => {
+  const n = Number(value);
+  return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback;
+};
+
+export function createReadGate(
+  env: Record<string, string | undefined> = process.env,
+  /** Whether a bearer token is a live readm3 credential: true, false (refused) or null (not ours). */
+  verifyToken: (token: string) => boolean | null = () => null,
+): ReadGate {
+  const apiKey = env.COINPAY_X402_KEY ?? "";
+  const payTo = env.CRAWL_PAY_TO ?? "";
+  const requests = number(env.READM3_FREE_READS, 20);
+  const windowSeconds = number(env.READM3_FREE_READS_WINDOW, 3600);
+  const gateway = createGateway({
+    siteUrl: (env.READM3_URL || "https://readm3.com").replace(/\/+$/, ""),
+    siteName: "readm3 reader",
+    ...(apiKey ? { coinpay: { apiKey } } : {}),
+    ...(payTo ? { payTo } : {}),
+    priceCents: number(env.READM3_READ_PRICE_CENTS, 100),
+    path: "/read/pass",
+    contact: "mailto:anthony@profullstack.com",
+    // Only the reader is metered here; the crawl gate looks after the pages.
+    isPaidAgent: () => false,
+    suspectAllowance: false,
+    freeQuota: { requests, windowSeconds, paths: READ_PATHS },
+    benefits: [
+      "Unlimited document reads for the day",
+      "PDFs, web pages, Word, Excel, PowerPoint, OpenDocument and EPUB",
+      "The same pass works on the API and the MCP",
+    ],
+    // A signed-in browser, verified: a made-up cookie must not open the meter.
+    exempt: (request: Request) => {
+      const session = (request.headers.get("cookie") ?? "").match(/(?:^|;\s*)(?:__Host-)?readm3_session=([^;]+)/)?.[1];
+      return Boolean(session && verifyToken(session) === true);
+    },
+    authenticate: (request: Request) => {
+      const header = request.headers.get("authorization") ?? "";
+      if (!header.startsWith("Bearer ")) return null;
+      const token = header.slice(7).trim();
+      // Power Keys and passes are the gateway's own business.
+      if (token.startsWith("pfs_") || token.includes(".")) return null;
+      return verifyToken(token);
+    },
+  });
+  return {
+    async handle(request: Request) {
+      const path = new URL(request.url).pathname;
+      // The sales page and its checkout belong to this gate as well.
+      if (!READ_PATHS.includes(path) && !path.startsWith("/read/pass")) return null;
+      return gateway.handle(request);
+    },
+  };
+}

server/read-mcp.ts

@@ -0,0 +1,109 @@
+/**
+ * readm3's hosted MCP server at https://readm3.com/mcp: stateless JSON-RPC over
+ * Streamable HTTP (POST, one JSON answer per request, no session), so any MCP
+ * client can add it by URL with no login. Its one tool is `read_url`, the
+ * document reader, metered by server/read-gate.ts like the REST endpoint.
+ */
+import { HttpError } from "./store.ts";
+import { DEFAULT_READ_CHARS, MAX_READ_CHARS, readUrl, type ReadOptions } from "./read.ts";
+
+const PROTOCOL = "2025-06-18";
+const SUPPORTED = new Set(["2024-11-05", "2025-03-26", "2025-06-18", "2025-11-25"]);
+
+export const READ_URL_TOOL = {
+  name: "read_url",
+  title: "Read a document at a URL",
+  description:
+    "Fetch a document by URL and return its text: PDFs (arXiv papers, reports), web pages, Word, Excel, PowerPoint, OpenDocument, EPUB, Markdown and other text. " +
+    "Use it to summarize, quote or answer questions about a link. Returns title, kind, page count, character count, whether the text was truncated, and the text. " +
+    `maxChars caps the text (default ${DEFAULT_READ_CHARS.toLocaleString("en-US")}, up to ${MAX_READ_CHARS.toLocaleString("en-US")}). ` +
+    "No account needed; a free hourly allowance, then an x402 reader pass.",
+  inputSchema: {
+    type: "object",
+    properties: {
+      url: { type: "string", description: "An http(s) URL on the public internet." },
+      maxChars: { type: "integer", minimum: 1000, maximum: MAX_READ_CHARS },
+    },
+    required: ["url"],
+    additionalProperties: false,
+  },
+  annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: true },
+};
+
+type Rpc = { jsonrpc?: string; id?: string | number | null; method?: string; params?: Record<string, unknown> };
+
+const cors = {
+  "access-control-allow-origin": "*",
+  "access-control-allow-methods": "POST, OPTIONS",
+  "access-control-allow-headers": "content-type, authorization, mcp-protocol-version, mcp-session-id, x-payment, payment-signature, x-crawl-pass, x-power-key",
+  "access-control-expose-headers": "payment-required, x-payment-response",
+  "cache-control": "no-store",
+};
+
+const reply = (id: Rpc["id"], result: unknown) => ({ jsonrpc: "2.0", id: id ?? null, result });
+const failure = (id: Rpc["id"], code: number, message: string) => ({ jsonrpc: "2.0", id: id ?? null, error: { code, message } });
+
+async function answer(message: Rpc, options: ReadOptions): Promise<unknown | null> {
+  const { id, method, params = {} } = message;
+  // A notification gets no answer.
+  if (id === undefined || id === null) return null;
+  switch (method) {
+    case "initialize": {
+      const asked = String(params.protocolVersion ?? "");
+      return reply(id, {
+        protocolVersion: SUPPORTED.has(asked) ? asked : PROTOCOL,
+        capabilities: { tools: { listChanged: false } },
+        serverInfo: { name: "readm3", title: "readm3 document reader", version: "1.0.0" },
+        instructions: "Call read_url with a link to get the document's text, then summarize or answer from it.",
+      });
+    }
+    case "ping":
+      return reply(id, {});
+    case "tools/list":
+      return reply(id, { tools: [READ_URL_TOOL] });
+    case "tools/call": {
+      if (params.name !== "read_url") return failure(id, -32602, `Unknown tool: ${String(params.name)}`);
+      const args = (params.arguments ?? {}) as Record<string, unknown>;
+      if (typeof args.url !== "string" || !args.url) return failure(id, -32602, "url is required.");
+      try {
+        const result = await readUrl(args.url, { ...options, maxChars: typeof args.maxChars === "number" ? args.maxChars : undefined });
+        const { text, ...about } = result;
+        return reply(id, {
+          content: [
+            { type: "text", text: `# ${result.title}\n\nSource: ${result.finalUrl} (${result.kind}${result.pages ? `, ${result.pages} pages` : ""}, ${result.chars.toLocaleString("en-US")} characters${result.truncated ? ", truncated" : ""})\n\n${text}` },
+          ],
+          structuredContent: { ...about, text },
+          isError: false,
+        });
+      } catch (error) {
+        const text = error instanceof Error ? error.message : "The document could not be read.";
+        return reply(id, { content: [{ type: "text", text }], isError: true });
+      }
+    }
+    default:
+      return failure(id, -32601, `Method not found: ${String(method)}`);
+  }
+}
+
+/** The /mcp route. Null for any other path. */
+export async function handleMcp(request: Request, options: ReadOptions = {}): Promise<Response | null> {
+  const url = new URL(request.url);
+  if (url.pathname !== "/mcp") return null;
+  if (request.method === "OPTIONS") return new Response(null, { status: 204, headers: cors });
+  if (request.method !== "POST")
+    return Response.json(failure(null, -32000, "POST JSON-RPC to this URL; there is no event stream."), { status: 405, headers: { ...cors, allow: "POST, OPTIONS" } });
+  let payload: unknown;
+  try {
+    const raw = await request.text();
+    if (raw.length > 64 * 1024) throw new HttpError(413, "Request too large.");
+    payload = JSON.parse(raw);
+  } catch {
+    return Response.json(failure(null, -32700, "Parse error"), { status: 400, headers: cors });
+  }
+  const batch = Array.isArray(payload);
+  const messages = (batch ? payload : [payload]) as Rpc[];
+  if (!messages.length || messages.length > 10) return Response.json(failure(null, -32600, "Invalid request"), { status: 400, headers: cors });
+  const answers = (await Promise.all(messages.map((m) => (m && typeof m === "object" ? answer(m, options) : Promise.resolve(failure(null, -32600, "Invalid request")))))).filter((a) => a !== null);
+  if (!answers.length) return new Response(null, { status: 202, headers: cors });
+  return Response.json(batch ? answers : answers[0], { headers: cors });
+}