diff --git a/README.md b/README.md index 84be148..38c8bb6 100644 --- a/README.md +++ b/README.md @@ -358,6 +358,31 @@ same scan (`level` is `"secret"`, `"personal"` or `null`, plus `findings`); the reports and never refuses, so a script decides for itself. A clean scan is not a guarantee: the check knows common shapes, not every secret. +## Read a document at a URL + +`readm3 read URL` prints the text of a PDF, a web page, a Word, Excel, PowerPoint, +OpenDocument or EPUB file, or any text file, so an agent or a script can summarize a link +without a PDF library of its own: + + readm3 read https://arxiv.org/pdf/2106.01820 + curl -s "https://readm3.com/api/v1/read?url=https://arxiv.org/pdf/2106.01820" + +The API is `GET /api/v1/read?url=` or `POST /api/v1/read {url, maxChars?}` and answers +`title`, `kind` (pdf, html, office, text), `pages`, `bytes`, `chars`, `truncated` and +`text` (100,000 characters by default, up to 400,000). Agents get it as the `read_url` +tool, both in `readm3 mcp` and on the hosted MCP server at **https://readm3.com/mcp** +(Streamable HTTP, no login, no session). + +No account is needed. Each address gets 20 reads an hour free; after that the answer is +`402 Payment Required` with an x402 offer for a $1 reader pass for the day (USDC through +CoinPay, sales page at `/read/pass`), which a pass or a Profullstack Power Key in +`Authorization: Bearer` or `x-crawl-pass` then skips. A signed-in account (session or API +token) is not metered. Only public addresses are fetched: private, loopback and +link-local hosts are refused on the first URL and on every redirect, and a document may +be up to 25 MB. A scanned PDF with no text layer is refused rather than guessed at. +Operators set `READM3_FREE_READS`, `READM3_FREE_READS_WINDOW` (seconds) and +`READM3_READ_PRICE_CENTS`; payments need `COINPAY_X402_KEY` and `CRAWL_PAY_TO`. + ## Editor Open https://readm3.com/viewer for a plain, fast text editor in your browser, with the diff --git a/bun.lock b/bun.lock index dfe7104..ff28434 100644 --- a/bun.lock +++ b/bun.lock @@ -15,6 +15,7 @@ "@profullstack/x402-gateway": "^0.16.1", "highlight.js": "^11.12.0", "marked": "^18.1.1", + "unpdf": "2.1.0", }, "devDependencies": { "@anthropic-ai/sdk": "^1.3.0", @@ -431,6 +432,8 @@ "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], + "unpdf": ["unpdf@2.1.0", "", { "peerDependencies": { "@napi-rs/canvas": "^0.1.69 || ^1.0.0" }, "optionalPeers": ["@napi-rs/canvas"] }, "sha512-G/Hsrm++g0EoubUhV8TDFItq6oQ+FUu9EMvIRtHUwseKXB0iJoCvaLvBGOpYUBqyvggv/XfxhYYtohL2QpE3xQ=="], + "unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="], "uuid": ["uuid@11.1.1", "", { "bin": { "uuid": "dist/esm/bin/uuid" } }, "sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ=="], diff --git a/desktop/package.json b/desktop/package.json index 415c5a4..73720a3 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -1,7 +1,7 @@ { "name": "readm3-desktop", "productName": "readm3", - "version": "0.31.0", + "version": "0.32.0", "private": true, "description": "readm3 as a desktop app: a plain, fast text editor for Markdown and code.", "author": "Profullstack, Inc. ", diff --git a/package.json b/package.json index 1889d0d..599abed 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/readm3", - "version": "0.31.0", + "version": "0.32.0", "description": "Markdown editing and sharing for people and teams. Web/PWA, CLI, API and MCP with version history, view/edit links, and a terminal reader.", "type": "module", "license": "MIT", @@ -51,7 +51,8 @@ "@profullstack/synconfig": "^0.1.3", "@profullstack/x402-gateway": "^0.16.1", "highlight.js": "^11.12.0", - "marked": "^18.1.1" + "marked": "^18.1.1", + "unpdf": "2.1.0" }, "devDependencies": { "@anthropic-ai/sdk": "^1.3.0", diff --git a/server/api.ts b/server/api.ts index 1e72d87..09e750c 100644 --- a/server/api.ts +++ b/server/api.ts @@ -7,6 +7,7 @@ import { createPaste, deletePaste, readPaste } from "./pastes.ts"; import { binaryType } from "../src/code.ts"; import type { Billing } from "./billing.ts"; import { PACKS_USD, type Payments } from "./payments.ts"; +import { readUrl } from "./read.ts"; /** Paid AI summaries and the CoinPay top-ups that fund them. */ export type Money = { billing: Billing; payments: Payments }; @@ -163,6 +164,19 @@ export function createApi( if (user) store.ensureWorkspace(user); const path = url.pathname.slice(8); if (path === "me" && request.method === "GET") return json({ user }); + // The document reader: GET ?url= or POST {url, maxChars}. No account needed; + // server/read-gate.ts meters callers before they get here. + if (path === "read") { + if (request.method !== "GET" && request.method !== "POST") throw new HttpError(405, "Method not allowed."); + limit(`read:${user?.id ?? ip}`, user ? 60 : 20); + const args: Record = request.method === "GET" + ? Object.fromEntries(url.searchParams) + : await body(request, 16 * 1024); + if (typeof args.url !== "string" || !args.url) throw new HttpError(400, "url is required."); + const maxChars = args.maxChars === undefined ? undefined : Number(args.maxChars); + if (maxChars !== undefined && !Number.isFinite(maxChars)) throw new HttpError(400, "maxChars must be a number."); + return json(await readUrl(args.url, { maxChars })); + } // The shared @profullstack/profile fields; private to the signed-in account. if (path === "me/profile") { const method = request.method as "GET" | "PUT" | "PATCH"; diff --git a/server/read-gate.ts b/server/read-gate.ts new file mode 100644 index 0000000..3c0be3c --- /dev/null +++ b/server/read-gate.ts @@ -0,0 +1,72 @@ +/** + * The meter on the document reader (`/api/v1/read` and the hosted MCP at `/mcp`). + * + * No account is needed: every caller gets READM3_FREE_READS reads per + * READM3_FREE_READS_WINDOW seconds (default 20 an hour, by address). Past that + * the answer is 402 with an x402 offer for a reader pass (READM3_READ_PRICE_CENTS, + * default $1 for a day), payable in USDC through CoinPay, rather than a 429. A pass + * or a Profullstack Power Key reads unmetered, and so does a signed-in readm3 + * account (session cookie or API token): accounts are throttled per minute by the + * API instead. Payments switch on only with COINPAY_X402_KEY and CRAWL_PAY_TO; + * without them the free allowance still applies and running out is still a 402. + */ +import { createGateway } from "@profullstack/x402-gateway"; + +export const READ_PATHS = ["/api/v1/read", "/mcp"]; + +export type ReadGate = { handle: (request: Request) => Promise }; + +const number = (value: string | undefined, fallback: number) => { + const n = Number(value); + return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback; +}; + +export function createReadGate( + env: Record = process.env, + /** Whether a bearer token is a live readm3 credential: true, false (refused) or null (not ours). */ + verifyToken: (token: string) => boolean | null = () => null, +): ReadGate { + const apiKey = env.COINPAY_X402_KEY ?? ""; + const payTo = env.CRAWL_PAY_TO ?? ""; + const requests = number(env.READM3_FREE_READS, 20); + const windowSeconds = number(env.READM3_FREE_READS_WINDOW, 3600); + const gateway = createGateway({ + siteUrl: (env.READM3_URL || "https://readm3.com").replace(/\/+$/, ""), + siteName: "readm3 reader", + ...(apiKey ? { coinpay: { apiKey } } : {}), + ...(payTo ? { payTo } : {}), + priceCents: number(env.READM3_READ_PRICE_CENTS, 100), + path: "/read/pass", + contact: "mailto:anthony@profullstack.com", + // Only the reader is metered here; the crawl gate looks after the pages. + isPaidAgent: () => false, + suspectAllowance: false, + freeQuota: { requests, windowSeconds, paths: READ_PATHS }, + benefits: [ + "Unlimited document reads for the day", + "PDFs, web pages, Word, Excel, PowerPoint, OpenDocument and EPUB", + "The same pass works on the API and the MCP", + ], + // A signed-in browser, verified: a made-up cookie must not open the meter. + exempt: (request: Request) => { + const session = (request.headers.get("cookie") ?? "").match(/(?:^|;\s*)(?:__Host-)?readm3_session=([^;]+)/)?.[1]; + return Boolean(session && verifyToken(session) === true); + }, + authenticate: (request: Request) => { + const header = request.headers.get("authorization") ?? ""; + if (!header.startsWith("Bearer ")) return null; + const token = header.slice(7).trim(); + // Power Keys and passes are the gateway's own business. + if (token.startsWith("pfs_") || token.includes(".")) return null; + return verifyToken(token); + }, + }); + return { + async handle(request: Request) { + const path = new URL(request.url).pathname; + // The sales page and its checkout belong to this gate as well. + if (!READ_PATHS.includes(path) && !path.startsWith("/read/pass")) return null; + return gateway.handle(request); + }, + }; +} diff --git a/server/read-mcp.ts b/server/read-mcp.ts new file mode 100644 index 0000000..1e1cc85 --- /dev/null +++ b/server/read-mcp.ts @@ -0,0 +1,109 @@ +/** + * readm3's hosted MCP server at https://readm3.com/mcp: stateless JSON-RPC over + * Streamable HTTP (POST, one JSON answer per request, no session), so any MCP + * client can add it by URL with no login. Its one tool is `read_url`, the + * document reader, metered by server/read-gate.ts like the REST endpoint. + */ +import { HttpError } from "./store.ts"; +import { DEFAULT_READ_CHARS, MAX_READ_CHARS, readUrl, type ReadOptions } from "./read.ts"; + +const PROTOCOL = "2025-06-18"; +const SUPPORTED = new Set(["2024-11-05", "2025-03-26", "2025-06-18", "2025-11-25"]); + +export const READ_URL_TOOL = { + name: "read_url", + title: "Read a document at a URL", + description: + "Fetch a document by URL and return its text: PDFs (arXiv papers, reports), web pages, Word, Excel, PowerPoint, OpenDocument, EPUB, Markdown and other text. " + + "Use it to summarize, quote or answer questions about a link. Returns title, kind, page count, character count, whether the text was truncated, and the text. " + + `maxChars caps the text (default ${DEFAULT_READ_CHARS.toLocaleString("en-US")}, up to ${MAX_READ_CHARS.toLocaleString("en-US")}). ` + + "No account needed; a free hourly allowance, then an x402 reader pass.", + inputSchema: { + type: "object", + properties: { + url: { type: "string", description: "An http(s) URL on the public internet." }, + maxChars: { type: "integer", minimum: 1000, maximum: MAX_READ_CHARS }, + }, + required: ["url"], + additionalProperties: false, + }, + annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: true }, +}; + +type Rpc = { jsonrpc?: string; id?: string | number | null; method?: string; params?: Record }; + +const cors = { + "access-control-allow-origin": "*", + "access-control-allow-methods": "POST, OPTIONS", + "access-control-allow-headers": "content-type, authorization, mcp-protocol-version, mcp-session-id, x-payment, payment-signature, x-crawl-pass, x-power-key", + "access-control-expose-headers": "payment-required, x-payment-response", + "cache-control": "no-store", +}; + +const reply = (id: Rpc["id"], result: unknown) => ({ jsonrpc: "2.0", id: id ?? null, result }); +const failure = (id: Rpc["id"], code: number, message: string) => ({ jsonrpc: "2.0", id: id ?? null, error: { code, message } }); + +async function answer(message: Rpc, options: ReadOptions): Promise { + const { id, method, params = {} } = message; + // A notification gets no answer. + if (id === undefined || id === null) return null; + switch (method) { + case "initialize": { + const asked = String(params.protocolVersion ?? ""); + return reply(id, { + protocolVersion: SUPPORTED.has(asked) ? asked : PROTOCOL, + capabilities: { tools: { listChanged: false } }, + serverInfo: { name: "readm3", title: "readm3 document reader", version: "1.0.0" }, + instructions: "Call read_url with a link to get the document's text, then summarize or answer from it.", + }); + } + case "ping": + return reply(id, {}); + case "tools/list": + return reply(id, { tools: [READ_URL_TOOL] }); + case "tools/call": { + if (params.name !== "read_url") return failure(id, -32602, `Unknown tool: ${String(params.name)}`); + const args = (params.arguments ?? {}) as Record; + if (typeof args.url !== "string" || !args.url) return failure(id, -32602, "url is required."); + try { + const result = await readUrl(args.url, { ...options, maxChars: typeof args.maxChars === "number" ? args.maxChars : undefined }); + const { text, ...about } = result; + return reply(id, { + content: [ + { type: "text", text: `# ${result.title}\n\nSource: ${result.finalUrl} (${result.kind}${result.pages ? `, ${result.pages} pages` : ""}, ${result.chars.toLocaleString("en-US")} characters${result.truncated ? ", truncated" : ""})\n\n${text}` }, + ], + structuredContent: { ...about, text }, + isError: false, + }); + } catch (error) { + const text = error instanceof Error ? error.message : "The document could not be read."; + return reply(id, { content: [{ type: "text", text }], isError: true }); + } + } + default: + return failure(id, -32601, `Method not found: ${String(method)}`); + } +} + +/** The /mcp route. Null for any other path. */ +export async function handleMcp(request: Request, options: ReadOptions = {}): Promise { + const url = new URL(request.url); + if (url.pathname !== "/mcp") return null; + if (request.method === "OPTIONS") return new Response(null, { status: 204, headers: cors }); + if (request.method !== "POST") + return Response.json(failure(null, -32000, "POST JSON-RPC to this URL; there is no event stream."), { status: 405, headers: { ...cors, allow: "POST, OPTIONS" } }); + let payload: unknown; + try { + const raw = await request.text(); + if (raw.length > 64 * 1024) throw new HttpError(413, "Request too large."); + payload = JSON.parse(raw); + } catch { + return Response.json(failure(null, -32700, "Parse error"), { status: 400, headers: cors }); + } + const batch = Array.isArray(payload); + const messages = (batch ? payload : [payload]) as Rpc[]; + if (!messages.length || messages.length > 10) return Response.json(failure(null, -32600, "Invalid request"), { status: 400, headers: cors }); + const answers = (await Promise.all(messages.map((m) => (m && typeof m === "object" ? answer(m, options) : Promise.resolve(failure(null, -32600, "Invalid request")))))).filter((a) => a !== null); + if (!answers.length) return new Response(null, { status: 202, headers: cors }); + return Response.json(batch ? answers : answers[0], { headers: cors }); +} diff --git a/server/read.ts b/server/read.ts new file mode 100644 index 0000000..d47b850 --- /dev/null +++ b/server/read.ts @@ -0,0 +1,271 @@ +/** + * Read a document at a URL and return its words: the reader behind + * `GET|POST /api/v1/read` and the hosted MCP's `read_url` tool. + * + * A PDF is read with unpdf (pdf.js, no dependencies of its own), a web page is + * cut down to its text, an office document or e-book goes through extract.ts, + * and anything textual (Markdown, JSON, CSV, code) comes back as it is. + * + * The fetch is the dangerous part, because the URL is whatever a stranger typed: + * only http(s) on the default or a high port, every hop's host resolved and + * refused when any address is private, loopback, link-local or otherwise not on + * the public internet, redirects followed by hand so each one is checked, and the + * body cut off at a size cap. + */ +import { lookup } from "node:dns/promises"; +import { isIP } from "node:net"; +import { HttpError } from "./store.ts"; +import { decodeEntities, documentText, extractable } from "./extract.ts"; + +/** The most a document may weigh: a long paper or a book chapter, not a dataset. */ +export const MAX_READ_BYTES = 25 * 1024 * 1024; +/** Characters returned by default, and the most a caller may ask for. */ +export const DEFAULT_READ_CHARS = 100_000; +export const MAX_READ_CHARS = 400_000; +const MAX_REDIRECTS = 5; +const TIMEOUT_MS = 30_000; + +export type ReadResult = { + url: string; + finalUrl: string; + title: string; + kind: "pdf" | "html" | "office" | "text"; + mime: string; + pages?: number; + bytes: number; + chars: number; + truncated: boolean; + text: string; +}; + +export type ReadOptions = { + maxChars?: number; + fetchImpl?: typeof fetch; + /** Every address a host name resolves to. Tests replace it. */ + resolve?: (host: string) => Promise; +}; + +const defaultResolve = async (host: string) => (await lookup(host, { all: true, verbatim: true })).map((a) => a.address); + +function v4Number(ip: string): number { + return ip.split(".").reduce((n, part) => n * 256 + Number(part), 0); +} +const V4_BLOCKED: [string, number][] = [ + ["0.0.0.0", 8], ["10.0.0.0", 8], ["100.64.0.0", 10], ["127.0.0.0", 8], ["169.254.0.0", 16], + ["172.16.0.0", 12], ["192.0.0.0", 24], ["192.0.2.0", 24], ["192.88.99.0", 24], ["192.168.0.0", 16], + ["198.18.0.0", 15], ["198.51.100.0", 24], ["203.0.113.0", 24], ["224.0.0.0", 4], ["240.0.0.0", 4], +]; + +/** Whether an address is somewhere a stranger's URL must never reach. */ +export function isPrivateAddress(ip: string): boolean { + const family = isIP(ip); + if (family === 4) { + const n = v4Number(ip); + return V4_BLOCKED.some(([base, bits]) => { + const mask = bits === 0 ? 0 : (~0 << (32 - bits)) >>> 0; + return ((n & mask) >>> 0) === ((v4Number(base) & mask) >>> 0); + }); + } + if (family === 6) { + const a = ip.toLowerCase().replace(/^\[|\]$/g, ""); + if (a === "::" || a === "::1") return true; + // IPv4-mapped and -translated addresses carry the v4 address that matters. + const mapped = a.match(/^(?:::ffff:(?:0:)?|64:ff9b::)(\d+\.\d+\.\d+\.\d+)$/); + if (mapped) return isPrivateAddress(mapped[1]); + if (/^::ffff:/.test(a)) return true; + const first = parseInt(a.split(":")[0] || "0", 16); + if ((first & 0xfe00) === 0xfc00) return true; // fc00::/7 unique local + if ((first & 0xffc0) === 0xfe80) return true; // fe80::/10 link local + if ((first & 0xff00) === 0xff00) return true; // multicast + if (first === 0x2001 && parseInt(a.split(":")[1] || "0", 16) === 0x0db8) return true; // documentation + return false; + } + return true; +} + +/** The URL, checked: http(s), no credentials, a public host on an ordinary port. */ +export async function checkUrl(raw: string, resolve = defaultResolve): Promise { + let url: URL; + try { + url = new URL(raw); + } catch { + throw new HttpError(400, "url must be an absolute http(s) URL."); + } + if (url.protocol !== "http:" && url.protocol !== "https:") throw new HttpError(400, "url must be an absolute http(s) URL."); + if (url.username || url.password) throw new HttpError(400, "A URL with a user name or password is not read."); + const port = url.port ? Number(url.port) : 0; + if (port && port !== 80 && port !== 443 && port < 1024) throw new HttpError(400, "That port is not read."); + const host = url.hostname.replace(/^\[|\]$/g, ""); + if (!host || host === "localhost" || host.endsWith(".localhost") || host.endsWith(".internal") || host.endsWith(".local")) + throw new HttpError(400, "Only public addresses are read."); + let addresses: string[]; + if (isIP(host)) addresses = [host]; + else { + try { + addresses = await resolve(host); + } catch { + throw new HttpError(422, `${host} does not resolve.`); + } + } + if (!addresses.length || addresses.some(isPrivateAddress)) throw new HttpError(400, "Only public addresses are read."); + return url; +} + +async function readCapped(response: Response, max: number): Promise { + if (Number(response.headers.get("content-length")) > max) + throw new HttpError(413, `The document is larger than ${Math.round(max / 1024 / 1024)} MB.`); + const reader = response.body?.getReader(); + if (!reader) return new Uint8Array(); + const chunks: Uint8Array[] = []; + let size = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + size += value.length; + if (size > max) { + await reader.cancel(); + throw new HttpError(413, `The document is larger than ${Math.round(max / 1024 / 1024)} MB.`); + } + chunks.push(value); + } + const bytes = new Uint8Array(size); + let at = 0; + for (const chunk of chunks) { bytes.set(chunk, at); at += chunk.length; } + return bytes; +} + +/** Fetch with each redirect checked like the first URL. */ +export async function fetchDocument(raw: string, { fetchImpl = fetch, resolve = defaultResolve }: ReadOptions = {}) { + let url = await checkUrl(raw, resolve); + const deadline = AbortSignal.timeout(TIMEOUT_MS); + for (let hop = 0; ; hop++) { + let response: Response; + try { + response = await fetchImpl(url, { + redirect: "manual", + signal: deadline, + headers: { + "user-agent": "readm3-reader/1.0 (+https://readm3.com/docs#read)", + accept: "application/pdf,text/html;q=0.9,text/markdown,text/plain;q=0.8,*/*;q=0.5", + }, + }); + } catch (error) { + const reason = error instanceof Error && error.name === "TimeoutError" ? "took too long" : "could not be reached"; + throw new HttpError(502, `${url.host} ${reason}.`); + } + if (response.status >= 300 && response.status < 400 && response.headers.get("location")) { + await response.body?.cancel(); + if (hop >= MAX_REDIRECTS) throw new HttpError(502, "Too many redirects."); + url = await checkUrl(new URL(response.headers.get("location")!, url).href, resolve); + continue; + } + if (!response.ok) { + await response.body?.cancel(); + throw new HttpError(502, `${url.host} answered ${response.status}.`); + } + const bytes = await readCapped(response, MAX_READ_BYTES); + return { url, bytes, mime: (response.headers.get("content-type") ?? "").split(";")[0].trim().toLowerCase() }; + } +} + +const BLOCK_TAGS = /<\/?(?:p|div|section|article|main|header|footer|li|ul|ol|tr|table|blockquote|pre|figure|figcaption|dd|dt|dl|br|hr)\b[^>]*>/gi; + +/** A web page's title and its readable words, as light Markdown. */ +export function htmlToText(html: string): { title: string; text: string } { + const pick = (re: RegExp) => decodeEntities((html.match(re)?.[1] ?? "").replace(/\s+/g, " ").trim()); + const title = pick(/]+property=["']og:title["'][^>]+content=["']([^"']*)["']/i) + || pick(/]*>([\s\S]*?)<\/title>/i); + // The article when the page marks one, else the body. + const main = html.match(/<(article|main)\b[^>]*>([\s\S]*?)<\/\1>/i)?.[2] + ?? html.match(/]*>([\s\S]*)<\/body>/i)?.[1] ?? html; + const text = main + .replace(//g, " ") + .replace(/<(script|style|noscript|svg|template|iframe|nav|aside|form|button|select)\b[\s\S]*?<\/\1>/gi, " ") + .replace(/]*>([\s\S]*?)<\/h\1>/gi, (_, level, inner) => `\n\n${"#".repeat(Number(level))} ${inner.replace(/<[^>]+>/g, " ").trim()}\n\n`) + .replace(/]*>/gi, "\n- ") + .replace(BLOCK_TAGS, "\n") + .replace(/<[^>]+>/g, " "); + return { + title, + text: decodeEntities(text) + .split("\n") + .map((line) => line.replace(/[ \t ]+/g, " ").trim()) + .join("\n") + .replace(/\n{3,}/g, "\n\n") + .trim(), + }; +} + +async function pdfText(bytes: Uint8Array): Promise<{ title: string; text: string; pages: number }> { + const { getDocumentProxy, extractText, getMeta } = await import("unpdf"); + let pdf; + try { + pdf = await getDocumentProxy(bytes); + } catch { + throw new HttpError(422, "The PDF could not be opened."); + } + try { + const meta = await getMeta(pdf).catch(() => null); + const { totalPages, text } = await extractText(pdf, { mergePages: false }); + const body = (text as string[]).map((page) => page.trim()).filter(Boolean).join("\n\n"); + const metaTitle = String((meta?.info as Record | undefined)?.Title ?? "").trim(); + // Many PDFs carry a file name ("Chapter 1_Schutz", "Microsoft Word - x.docx") as their title. + const useful = metaTitle && !/[_]|\.(docx?|tex|pdf)$|^untitled$|^microsoft /i.test(metaTitle); + const firstLine = body.split("\n").find((line) => line.trim().length > 3)?.trim() ?? ""; + return { title: useful ? metaTitle : firstLine.slice(0, 200), text: body, pages: totalPages }; + } finally { + await (pdf as unknown as { destroy?: () => Promise }).destroy?.(); + } +} + +const TEXT_MIME = /^(text\/|application\/(json|xml|x-yaml|yaml|javascript|ld\+json|rss\+xml|atom\+xml|x-ndjson|toml))/; + +/** Fetch `raw` and return its text, cut to `maxChars`. */ +export async function readUrl(raw: string, options: ReadOptions = {}): Promise { + const maxChars = Math.min(Math.max(Math.floor(options.maxChars ?? DEFAULT_READ_CHARS), 1_000), MAX_READ_CHARS); + const { url, bytes, mime } = await fetchDocument(raw, options); + // pdf.js takes ownership of the buffer and detaches it, so measure first. + const size = bytes.length; + const path = decodeURIComponent(url.pathname); + const head = new TextDecoder().decode(bytes.subarray(0, 512)).trimStart().toLowerCase(); + let kind: ReadResult["kind"]; + let title = ""; + let text: string; + let pages: number | undefined; + if (mime === "application/pdf" || head.startsWith("%pdf-")) { + ({ title, text, pages } = await pdfText(bytes)); + kind = "pdf"; + } else if (mime === "text/html" || mime === "application/xhtml+xml" || head.startsWith(" maxChars; + return { + url: raw, + finalUrl: url.href, + title, + kind, + mime: mime || (kind === "pdf" ? "application/pdf" : "text/plain"), + ...(pages ? { pages } : {}), + bytes: size, + chars: text.length, + truncated, + text: truncated ? text.slice(0, maxChars) : text, + }; +} diff --git a/server/test/read.test.ts b/server/test/read.test.ts new file mode 100644 index 0000000..6b3b679 --- /dev/null +++ b/server/test/read.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, test } from "bun:test"; +import { checkUrl, htmlToText, isPrivateAddress, readUrl } from "../read.ts"; +import { createReadGate } from "../read-gate.ts"; +import { handleMcp } from "../read-mcp.ts"; + +/** A one-page PDF whose page says `words`, with correct xref offsets. */ +function tinyPdf(words: string): Uint8Array { + const stream = `BT /F1 18 Tf 72 720 Td (${words}) Tj ET`; + const objects = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>", + `<< /Length ${stream.length} >>\nstream\n${stream}\nendstream`, + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ]; + let out = "%PDF-1.4\n"; + const offsets: number[] = []; + objects.forEach((body, i) => { offsets.push(out.length); out += `${i + 1} 0 obj\n${body}\nendobj\n`; }); + const xref = out.length; + out += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n${offsets.map((o) => `${String(o).padStart(10, "0")} 00000 n \n`).join("")}`; + out += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`; + return new TextEncoder().encode(out); +} + +const publicDns = async () => ["93.184.216.34"]; +const serve = (routes: Record Response>): typeof fetch => + (async (input: RequestInfo | URL) => { + const key = new URL(String(input instanceof Request ? input.url : input)).href; + const route = routes[key]; + return route ? route() : new Response("missing", { status: 404 }); + }) as typeof fetch; + +describe("address checks", () => { + test("private, loopback, link-local and mapped addresses are refused", () => { + for (const ip of ["127.0.0.1", "10.1.2.3", "172.20.0.1", "192.168.1.1", "169.254.169.254", "100.64.0.1", "0.0.0.0", "::1", "fd00::1", "fe80::1", "::ffff:127.0.0.1", "::ffff:7f00:1"]) + expect(isPrivateAddress(ip)).toBe(true); + for (const ip of ["93.184.216.34", "8.8.8.8", "2606:4700::1111"]) expect(isPrivateAddress(ip)).toBe(false); + }); + test("URLs aimed inside are refused before any fetch", async () => { + await expect(checkUrl("http://127.0.0.1/")).rejects.toThrow("public"); + await expect(checkUrl("http://localhost:3000/")).rejects.toThrow("public"); + await expect(checkUrl("file:///etc/passwd")).rejects.toThrow("http(s)"); + await expect(checkUrl("http://user:pw@example.com/")).rejects.toThrow("password"); + await expect(checkUrl("http://sneaky.example/", async () => ["93.184.216.34", "10.0.0.5"])).rejects.toThrow("public"); + await expect(checkUrl("https://example.com:22/", publicDns)).rejects.toThrow("port"); + expect((await checkUrl("https://example.com/a.pdf", publicDns)).href).toBe("https://example.com/a.pdf"); + }); + test("a redirect into the private network is refused", async () => { + const fetchImpl = serve({ "https://example.com/go": () => new Response(null, { status: 302, headers: { location: "http://169.254.169.254/latest/meta-data/" } }) }); + await expect(readUrl("https://example.com/go", { fetchImpl, resolve: publicDns })).rejects.toThrow("public"); + }); +}); + +describe("reading", () => { + test("a PDF comes back as its text", async () => { + const pdf = tinyPdf("Intuition in Einsteinian Physics"); + const fetchImpl = serve({ "https://arxiv.org/pdf/2106.01820": () => new Response(pdf, { headers: { "content-type": "application/pdf" } }) }); + const read = await readUrl("https://arxiv.org/pdf/2106.01820", { fetchImpl, resolve: publicDns }); + expect(read.kind).toBe("pdf"); + expect(read.pages).toBe(1); + expect(read.bytes).toBe(pdf.length); + expect(read.text).toContain("Intuition in Einsteinian Physics"); + expect(read.title).toContain("Intuition"); + }); + test("a web page keeps its words and drops scripts and navigation", async () => { + const html = `A & B

Heading

First paragraph.

  • one
  • two
`; + const { title, text } = htmlToText(html); + expect(title).toBe("A & B"); + expect(text).toContain("# Heading"); + expect(text).toContain("First paragraph."); + expect(text).toContain("- one"); + expect(text).not.toContain("evil"); + expect(text).not.toContain("Home Docs"); + }); + test("redirects are followed, text is capped, binaries are refused", async () => { + const fetchImpl = serve({ + "https://example.com/old": () => new Response(null, { status: 301, headers: { location: "/notes.md" } }), + "https://example.com/notes.md": () => new Response("# Notes\n\n" + "x".repeat(5000), { headers: { "content-type": "text/markdown" } }), + "https://example.com/bin": () => new Response(new Uint8Array([1, 0, 2, 0, 3]), { headers: { "content-type": "application/octet-stream" } }), + "https://example.com/img": () => new Response("GIF89a", { headers: { "content-type": "image/gif" } }), + }); + const read = await readUrl("https://example.com/old", { fetchImpl, resolve: publicDns, maxChars: 1000 }); + expect(read.finalUrl).toBe("https://example.com/notes.md"); + expect(read.truncated).toBe(true); + expect(read.text.length).toBe(1000); + await expect(readUrl("https://example.com/bin", { fetchImpl, resolve: publicDns })).rejects.toThrow("binary"); + await expect(readUrl("https://example.com/img", { fetchImpl, resolve: publicDns })).rejects.toThrow("not read"); + }); +}); + +describe("read gate", () => { + const env = { COINPAY_X402_KEY: "cp_live_test", CRAWL_PAY_TO: "0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5", READM3_FREE_READS: "2" }; + const ask = (gate: ReturnType, path: string, headers: Record = {}) => + gate.handle(new Request(`https://readm3.com${path}`, { method: "POST", headers: { "x-real-ip": "203.0.113.9", ...headers } })); + + test("free reads, then a 402 with an x402 offer", async () => { + const gate = createReadGate(env); + expect(await ask(gate, "/api/v1/read")).toBeNull(); + expect(await ask(gate, "/mcp")).toBeNull(); + const sold = await ask(gate, "/api/v1/read"); + expect(sold?.status).toBe(402); + expect(sold?.headers.get("payment-required")).toBeTruthy(); + }); + test("other paths are not this gate's business", async () => { + const gate = createReadGate({ ...env, READM3_FREE_READS: "1" }); + for (let i = 0; i < 5; i++) expect(await ask(gate, "/api/v1/pastes")).toBeNull(); + }); + test("a verified account is never metered; a made-up cookie or token is", async () => { + const gate = createReadGate({ ...env, READM3_FREE_READS: "1" }, (token) => token === "good"); + for (let i = 0; i < 3; i++) expect(await ask(gate, "/api/v1/read", { authorization: "Bearer good" })).toBeNull(); + for (let i = 0; i < 3; i++) expect(await ask(gate, "/api/v1/read", { cookie: "__Host-readm3_session=good" })).toBeNull(); + expect((await ask(gate, "/api/v1/read", { authorization: "Bearer bad" }))?.status).toBe(401); + expect(await ask(gate, "/api/v1/read", { cookie: "__Host-readm3_session=fake", "x-real-ip": "198.51.100.7" })).toBeNull(); + expect((await ask(gate, "/api/v1/read", { cookie: "__Host-readm3_session=fake", "x-real-ip": "198.51.100.7" }))?.status).toBe(402); + }); +}); + +describe("hosted MCP", () => { + const rpc = (body: unknown) => handleMcp(new Request("https://readm3.com/mcp", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }), { resolve: publicDns, fetchImpl: serve({ "https://example.com/a.md": () => new Response("# Hello\n\nworld", { headers: { "content-type": "text/markdown" } }) }) }); + + test("initialize, list and call read_url", async () => { + const init = (await (await rpc({ jsonrpc: "2.0", id: 1, method: "initialize", params: { protocolVersion: "2025-06-18" } }))!.json()) as { result: { protocolVersion: string } }; + expect(init.result.protocolVersion).toBe("2025-06-18"); + expect((await rpc({ jsonrpc: "2.0", method: "notifications/initialized" }))!.status).toBe(202); + const list = (await (await rpc({ jsonrpc: "2.0", id: 2, method: "tools/list" }))!.json()) as { result: { tools: Array<{ name: string }> } }; + expect(list.result.tools.map((t) => t.name)).toEqual(["read_url"]); + const call = (await (await rpc({ jsonrpc: "2.0", id: 3, method: "tools/call", params: { name: "read_url", arguments: { url: "https://example.com/a.md" } } }))!.json()) as { result: { isError: boolean; content: Array<{ text: string }> } }; + expect(call.result.isError).toBe(false); + expect(call.result.content[0]!.text).toContain("world"); + }); + test("a refused URL is a tool error, not a crash", async () => { + const call = (await (await rpc({ jsonrpc: "2.0", id: 4, method: "tools/call", params: { name: "read_url", arguments: { url: "http://127.0.0.1/" } } }))!.json()) as { result: { isError: boolean } }; + expect(call.result.isError).toBe(true); + }); +}); diff --git a/site/build.ts b/site/build.ts index a5fefb5..84ce7dc 100644 --- a/site/build.ts +++ b/site/build.ts @@ -432,6 +432,14 @@ inline code, links, fenced and indented code blocks, nested and task lists, bloc tables with per-column alignment, rules, images as labelled placeholders, and YAML front matter. +## Read a document at a URL + +\`GET ${SITE}/api/v1/read?url=\` (or POST {url, maxChars}) returns the text of a PDF, +web page, office document, EPUB or text file as JSON: title, kind, pages, chars, truncated, +text. The hosted MCP server at ${SITE}/mcp (Streamable HTTP, no login) has one tool, +read_url. No account: 20 reads an hour per address, then 402 with an x402 offer for a $1 +day pass (${SITE}/read/pass). The CLI is \`readm3 read URL\`. + ## Install curl -fsSL ${SITE}/install.sh | sh diff --git a/site/server.ts b/site/server.ts index 03fd8cb..9576207 100644 --- a/site/server.ts +++ b/site/server.ts @@ -21,6 +21,8 @@ import { createSyncApi } from "../server/sync-api.ts"; import { Blobs, createBlobApi } from "../server/blobs.ts"; import { FileHistory, createFileHistoryApi } from "../server/file-history.ts"; import { createCrawlGate, type CrawlGate } from "../server/crawl-gate.ts"; +import { createReadGate, type ReadGate } from "../server/read-gate.ts"; +import { handleMcp } from "../server/read-mcp.ts"; const dist = join(dirname(fileURLToPath(import.meta.url)), "dist"); const port = Number(process.env.PORT ?? 3000); @@ -113,7 +115,7 @@ function cacheFor(path: string): string { return "public, max-age=300"; } -export function serveSite(options: { accounts?: Accounts; coinpay?: ConstructorParameters[2]; port?: number; hostname?: string; summarize?: Summarize | null; speak?: Speak | null; payments?: PaymentOptions; blobDir?: string; crawlGate?: CrawlGate | null } = {}) { +export function serveSite(options: { accounts?: Accounts; coinpay?: ConstructorParameters[2]; port?: number; hostname?: string; summarize?: Summarize | null; speak?: Speak | null; payments?: PaymentOptions; blobDir?: string; crawlGate?: CrawlGate | null; readGate?: ReadGate | null } = {}) { const listenPort = options.port ?? port; const database = process.env.READM3_DB || join(dirname(dist), "..", "data", "readm3.sqlite"); const accounts = options.accounts ?? new Accounts( @@ -162,6 +164,10 @@ export function serveSite(options: { accounts?: Accounts; coinpay?: ConstructorP // `readm3 login`: OAuth 2.1 for the CLI (server/oauth.ts). const oauth = createOAuth(accounts); const crawlGate = options.crawlGate !== undefined ? options.crawlGate : createCrawlGate(); + // The document reader's meter: a free allowance, then an x402 pass (server/read-gate.ts). + const readGate = options.readGate !== undefined ? options.readGate : createReadGate(process.env, (token) => { + try { return store.authenticate(token) ? true : false; } catch { return false; } + }); return Bun.serve({ port: listenPort, hostname: options.hostname ?? "0.0.0.0", @@ -194,6 +200,22 @@ export function serveSite(options: { accounts?: Accounts; coinpay?: ConstructorP if (sold) return sold; } + // The reader (REST and MCP). On /mcp only a tool call is metered: initialize and + // tools/list cost nothing, so a client's handshake never eats the allowance. + if (readGate && (url.pathname === "/api/v1/read" || url.pathname === "/mcp" || url.pathname.startsWith("/read/pass"))) { + let metered = true; + if (url.pathname === "/mcp") { + const peek = request.method === "POST" ? await request.clone().text().catch(() => "") : ""; + metered = /"tools\/call"/.test(peek); + } + if (metered) { + const sold = await readGate.handle(request); + if (sold) return sold; + } + const mcp = await handleMcp(request); + if (mcp) return mcp; + } + // Railway overwrites X-Real-IP at the edge; never trust client-supplied X-Forwarded-For. const ip = process.env.RAILWAY_ENVIRONMENT_ID ? request.headers.get("x-real-ip") || server.requestIP(request)?.address || "unknown" diff --git a/src/cli.ts b/src/cli.ts index cdac567..d033ce1 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -12,7 +12,7 @@ import { run, type ViewerOptions } from "./viewer.ts"; import { isCloudCommand, cloudMain, CLOUD_USAGE } from "./cloud-cli.ts"; import { isDesktopCommand, desktopMain, installedDesktop, prefersDesktop, DESKTOP_USAGE } from "./desktop-cli.ts"; -export const VERSION = "0.31.0"; +export const VERSION = "0.32.0"; /** The help text, exported so a test can hold it to the flags it documents. */ export const USAGE = `readm3 — a plain text editor for Markdown and code diff --git a/src/cloud-cli.ts b/src/cloud-cli.ts index 4f901e9..5075ac0 100644 --- a/src/cloud-cli.ts +++ b/src/cloud-cli.ts @@ -41,6 +41,7 @@ export const CLOUD_USAGE = `Shared Markdown readm3 orgs list | create --name NAME | update ID --name NAME | delete ID readm3 teams list --org ID | create --org ID --name NAME readm3 members list --org ID + readm3 read URL [--json] [--max-chars N] Print the text of a PDF, web page or document readm3 cloud OPERATION --args '{"key":"value"}' readm3 mcp Start the stdio MCP server @@ -59,6 +60,7 @@ const commands = new Set([ "docs", "shared", "paste", + "read", "orgs", "teams", "members", @@ -179,6 +181,14 @@ export async function cloudMain(argv: string[]) { else process.stdout.write(paste.url + "\n"); return; } + if (command === "read") { + if (!positional[0]) throw new Error("A URL is required: readm3 read https://arxiv.org/pdf/2106.01820"); + const maxChars = flags["max-chars"] ? Number(flags["max-chars"]) : undefined; + const read = await cloudRequest<{ title: string; finalUrl: string; text: string; truncated: boolean }>("read", "POST", { url: positional[0], maxChars }, cloudConfig()); + if (flags.json) output(read); + else process.stdout.write(`# ${read.title}\n\n${read.text}\n${read.truncated ? "\n[truncated; pass --max-chars for more]\n" : ""}`); + return; + } if (command === "logout") { // A `readm3 login` sign-in is revoked on the server; a pasted /admin token is only forgotten. const signedIn = await oauthStore().load(); diff --git a/src/mcp.ts b/src/mcp.ts index 124c0b7..1876d44 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -12,6 +12,7 @@ const role = { type: "string", enum: ["view", "edit"] }; const definitions: [string, string, Record, string[]][] = [ ["paste_create", "Create an anonymous paste: any text file behind a secret link, no account needed. Markdown renders as a document; code (JSON, JavaScript, Python, YAML, SQL and 30 more) is highlighted with folding. The language is detected from the title's extension, else from the content; pass language to force one. Expires after 7 days by default (1h, 1d, 7d or 30d), 256 KB at most. Returns the URL and a raw URL; keep them, the server cannot show them again. The result's sensitive field lists any key, token, password, email address, public IP or named home directory in what the link exposes (secrets masked); when its level is \"secret\", tell the user before passing the link on.", { source: string, title: { type: "string", description: "A file name; its extension decides the language." }, language: { type: "string", description: "A highlight.js name such as json, javascript, python, bash, yaml or markdown. Detected when omitted." }, expiresIn: { type: "string", enum: ["1h", "1d", "7d", "30d"] } }, ["source"]], ["table_query", "Read a CSV or TSV the way the readm3 table view does, without loading it all into context. Give exactly one of source (the text), path (a local file) or url (a readm3 paste link or a raw http(s) file URL). The delimiter, header row and column types (number, date, boolean, text) are detected. filter keeps rows matching every space-separated term: free text, col:text, col=v, col!=v, col>n, col>=n, col