From 70580b760c042d7a03f5b9c43e51fbbfd4b1c850 Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sat, 26 Sep 2026 15:54:58 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=A4=96=20Serve=20the=20documents=20agents?= =?UTF-8?q?=20read=20as=20markdown?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit llms.txt led agents to markdown for nothing it pointed at: the contract and the guides came from raw.githubusercontent.com, pinned to whatever the v4 branch happened to be, and the packages and the API came as rendered pages. Serve all four from the site instead, each from the same source its page is built from: /AGENTS.md docs/agents.md, with its canonical links rewritten to whichever site is serving /guides/:series/:id.md the guide's mdx source /x/:workspacePath.md the package readme, plus the npm install command unless the readme already gives it /api.md the symbols the HTML index lists /api/:series/:symbol.md the declaration, rendered through the same Type component the page uses, its documentation, and where the code lives Each markdown route is registered before the page it shadows, so `.md` reads as a suffix rather than part of an id, and each lists its documents in the sitemap so a static build captures them. An unknown id answers 404 rather than throwing. The three response shapes go through one helper, which also settles the cache header: `no-cache`, because it only ever reaches a browser talking to the dev server, and these documents differ per environment. The smoke tests in www/tests exercise the routes against a running site. They are not part of the unit suite, which has no server; the www workflow runs them against the one it starts before a static build. --- .github/workflows/www.yaml | 14 +++ AGENTS.md | 10 +- www/components/footer.tsx | 5 +- www/deno.json | 4 +- www/lib/api-markdown.ts | 82 ++++++++++++++ www/lib/markdown-response.ts | 28 +++++ www/main.tsx | 23 ++++ www/resources/guides.ts | 8 +- www/routes/agents-md-route.ts | 57 ++++++++++ www/routes/api-markdown-route.ts | 142 +++++++++++++++++++++++++ www/routes/guides-markdown-route.ts | 57 ++++++++++ www/routes/llms-txt-route.ts | 41 ++++--- www/routes/x-package-markdown-route.ts | 64 +++++++++++ www/tests/markdown-routes.ts | 120 +++++++++++++++++++++ 14 files changed, 629 insertions(+), 26 deletions(-) create mode 100644 www/lib/api-markdown.ts create mode 100644 www/lib/markdown-response.ts create mode 100644 www/routes/agents-md-route.ts create mode 100644 www/routes/api-markdown-route.ts create mode 100644 www/routes/guides-markdown-route.ts create mode 100644 www/routes/x-package-markdown-route.ts create mode 100644 www/tests/markdown-routes.ts diff --git a/.github/workflows/www.yaml b/.github/workflows/www.yaml index 7ac6a9d86..13a7e7c4c 100644 --- a/.github/workflows/www.yaml +++ b/.github/workflows/www.yaml @@ -68,6 +68,13 @@ jobs: timeout-minutes: 5 working-directory: ./www + - name: Smoke test + run: deno task smoke + env: + SMOKE_URL: http://127.0.0.1:8000 + timeout-minutes: 5 + working-directory: ./www + - name: Download Staticalize run: | wget https://github.com/thefrontside/staticalize/releases/download/v0.3.1/staticalize-linux.tar.gz \ @@ -171,6 +178,13 @@ jobs: timeout-minutes: 5 working-directory: ./www + - name: Smoke test + run: deno task smoke + env: + SMOKE_URL: http://127.0.0.1:8000 + timeout-minutes: 5 + working-directory: ./www + - name: Download Staticalize run: | wget https://github.com/thefrontside/staticalize/releases/download/v0.3.1/staticalize-linux.tar.gz \ diff --git a/AGENTS.md b/AGENTS.md index e50877c5a..3ce038e73 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,10 +5,12 @@ repository-only rules on top of the public behavioral contract, which it does not repeat. Before you modify or reason about Effection code, read the public contract in -[`docs/agents.md`](docs/agents.md), the copy checked out on this branch. It -holds the invariants: operations versus promises, scope ownership, tasks and -halting, context, the concurrency operations, promise interoperability, -resources and cleanup, `ensure()`, `useAbortSignal()`, and streams. +[`docs/agents.md`](docs/agents.md), the copy checked out on this branch. The +website publishes the same document at +. It holds the invariants: operations +versus promises, scope ownership, tasks and halting, context, the concurrency +operations, promise interoperability, resources and cleanup, `ensure()`, +`useAbortSignal()`, and streams. Instructions scoped to a subdirectory, such as [`www/AGENTS.md`](www/AGENTS.md) for the website, apply in addition to this file. diff --git a/www/components/footer.tsx b/www/components/footer.tsx index c01a7801c..2aeab9a93 100644 --- a/www/components/footer.tsx +++ b/www/components/footer.tsx @@ -38,10 +38,7 @@ export function Footer(): JSX.Element { llms.txt - + AGENTS.md diff --git a/www/deno.json b/www/deno.json index 173d8dd02..a3556e421 100644 --- a/www/deno.json +++ b/www/deno.json @@ -2,7 +2,8 @@ "tasks": { "dev": "deno run -A @effectionx/watch deno run -A main.tsx", "staticalize": "deno run -A npm:staticalize@0.3.1 --site http://localhost:8000 --output=built --base=http://localhost:8000", - "test": "deno test --allow-run --allow-write --allow-read --allow-env" + "test": "deno test --allow-run --allow-write --allow-read --allow-env", + "smoke": "deno test --allow-net --allow-env tests/markdown-routes.ts" }, "lint": { "exclude": [ @@ -56,6 +57,7 @@ "hast-util-from-html": "npm:hast-util-from-html@2.0.3", "hast-util-shift-heading": "npm:hast-util-shift-heading@4.0.0", "hast-util-to-html": "npm:hast-util-to-html@9.0.0", + "hast-util-to-text": "npm:hast-util-to-text@4.0.2", "mdast": "npm:mdast@^3.0.0", "mdx": "npm:mdx@^0.3.1", "octokit": "npm:octokit@4.0.3", diff --git a/www/lib/api-markdown.ts b/www/lib/api-markdown.ts new file mode 100644 index 000000000..2685618fe --- /dev/null +++ b/www/lib/api-markdown.ts @@ -0,0 +1,82 @@ +/** + * The markdown twin of the API reference: the same symbols the HTML index and + * symbol pages show, written out as markdown so that an agent following + * `llms.txt` never has to read a rendered page. + * + * These are the string builders. The routes in `routes/api-markdown-route.ts` + * supply the content, which comes from the same `pkg.docs()` the HTML routes + * render. + */ + +import { all, type Operation } from "effection"; + +import { url } from "../context/url.ts"; + +export interface ApiSymbol { + name: string; + /** exported from the package's `./experimental` entrypoint */ + experimental?: boolean; +} + +export interface ApiVersion { + /** series the symbols belong to, e.g. `v4` */ + series: string; + /** version of the release that series resolves to, e.g. `4.1.1` */ + version: string; + symbols: ApiSymbol[]; +} + +export interface ApiSection { + /** the declaration, as the symbol page shows it */ + signature: string; + /** the symbol's documentation */ + markdown: string; + /** where the declaration lives */ + source?: string; +} + +/** + * Path of a symbol's markdown page. Experimental symbols live under an + * `/experimental` segment, the same as their HTML pages. + */ +export function apiSymbolPath(series: string, symbol: ApiSymbol): string { + let namespace = symbol.experimental ? `${series}/experimental` : series; + + return `/api/${namespace}/${symbol.name}.md`; +} + +export function* apiIndexMarkdown( + versions: ApiVersion[], +): Operation { + let sections = yield* all( + versions.map(function* ({ series, version, symbols }) { + let entries = yield* all(symbols.map(function* (symbol) { + let href = yield* url(apiSymbolPath(series, symbol)); + let suffix = symbol.experimental ? " (experimental)" : ""; + + return `- [${symbol.name}](${href})${suffix}`; + })); + + return [`## ${version}`, "", ...entries].join("\n"); + }), + ); + + return `${["# API Reference", ...sections].join("\n\n")}\n`; +} + +export function apiSymbolMarkdown( + name: string, + sections: ApiSection[], +): string { + let bodies = sections.map(({ signature, markdown, source }) => { + let body = ["```ts", signature, "```", "", markdown.trim()]; + + if (source) { + body.push("", `[View code](${source})`); + } + + return body.join("\n"); + }); + + return `${[`# ${name}`, ...bodies].join("\n\n")}\n`; +} diff --git a/www/lib/markdown-response.ts b/www/lib/markdown-response.ts new file mode 100644 index 000000000..0331a5d8a --- /dev/null +++ b/www/lib/markdown-response.ts @@ -0,0 +1,28 @@ +/** + * Responses for the markdown the site serves to agents: the behavioral + * contract, the guides, and package readmes. + * + * `no-cache` rather than a max age because the header only ever reaches a + * browser talking to the dev server — a static build copies the body and + * netlify supplies its own — and these documents are rewritten per + * environment, so a browser must not hold one environment's copy and show it + * in another. The etag plugin answers the revalidation with a 304. + */ +export function markdown(content: string): Response { + return new Response(content, { + headers: { + "Content-Type": "text/markdown; charset=utf-8", + "Cache-Control": "no-cache", + }, + }); +} + +export function notFound(message: string): Response { + return new Response(`${message}\n`, { + status: 404, + headers: { + "Content-Type": "text/plain; charset=utf-8", + "Cache-Control": "no-cache", + }, + }); +} diff --git a/www/main.tsx b/www/main.tsx index b1f846a0f..45004e652 100644 --- a/www/main.tsx +++ b/www/main.tsx @@ -37,6 +37,13 @@ import { redirectIndexRoute } from "./routes/redirect-index-route.tsx"; import { searchRoute } from "./routes/search-route.tsx"; import { initClones } from "./lib/clones.ts"; import { initOctokitContext } from "./lib/octokit.ts"; +import { guidesMarkdownRoute } from "./routes/guides-markdown-route.ts"; +import { xPackageMarkdownRoute } from "./routes/x-package-markdown-route.ts"; +import { + apiIndexMarkdownRoute, + apiSymbolMarkdownRoute, +} from "./routes/api-markdown-route.ts"; +import { agentsMdRoute } from "./routes/agents-md-route.ts"; import { currentRequestPlugin } from "./plugins/current-request.ts"; import { verboseLogging } from "./context/logging.ts"; @@ -97,12 +104,27 @@ function* serve(options: Options) { ...stableSeries.map((s) => route(`/guides/${s.name}`, redirectIndexRoute(firstPage(s.name))) ), + // before the page route, so that `.md` is a suffix and not a guide id + route("/guides/:series/:id.md", guidesMarkdownRoute()), route("/guides/:series/:id", guidesRoute({ search: true })), route("/contrib", xIndexRedirect()), route("/contrib/:workspacePath", xPackageRedirect()), route("/x", xIndexRoute({ search: true })), + // before the page route, so that `.md` is a suffix and not a package + route("/x/:workspacePath.md", xPackageMarkdownRoute()), route("/x/:workspacePath", xPackageRoute({ search: true })), route("/api", apiIndexRoute({ search: true })), + // before the page routes, so that `.md` is a suffix and not a symbol + route("/api.md", apiIndexMarkdownRoute()), + ...series.map((s) => + route(`/api/${s.name}/:symbol.md`, apiSymbolMarkdownRoute(s.name)) + ), + ...series.map((s) => + route( + `/api/${s.name}/experimental/:symbol.md`, + apiSymbolMarkdownRoute(s.name, { entrypoint: "./experimental" }), + ) + ), // API docs for all series including prereleases ...series.map((s) => route( @@ -124,6 +146,7 @@ function* serve(options: Options) { route("/blog", blogIndexRoute({ search: true })), route("/blog/feed.xml", blogFeedRoute()), route("/llms.txt", llmsTxtRoute()), + route("/AGENTS.md", agentsMdRoute()), route("/blog/tags/:tag", blogTagRoute({ search: true })), route("/blog/:id", blogPostRoute({ search: true })), route("/blog/:id/:name.png", blogImageRoute()), diff --git a/www/resources/guides.ts b/www/resources/guides.ts index f5ab0ef33..b093d00cd 100644 --- a/www/resources/guides.ts +++ b/www/resources/guides.ts @@ -1,4 +1,4 @@ -import { basename } from "@std/path"; +import { basename, toFileUrl } from "@std/path"; import { all, createContext, @@ -98,7 +98,11 @@ export function loadGuides(dirpath: string): Operation { let loaders = new Map>(); let structureModule = yield* until( - import(`${dirpath}/docs/structure.json`, { with: { type: "json" } }), + // a path is not a module specifier on windows, where it starts with a + // drive letter that deno reads as an unsupported scheme + import(toFileUrl(`${dirpath}/docs/structure.json`).href, { + with: { type: "json" }, + }), ); let structure = Structure.parse(structureModule.default); diff --git a/www/routes/agents-md-route.ts b/www/routes/agents-md-route.ts new file mode 100644 index 000000000..0ad9663c7 --- /dev/null +++ b/www/routes/agents-md-route.ts @@ -0,0 +1,57 @@ +import type { Operation } from "effection"; +import { until } from "effection"; +import { fromFileUrl } from "@std/path"; + +import type { SitemapRoute } from "../plugins/sitemap.ts"; +import { url } from "../context/url.ts"; +import { markdown } from "../lib/markdown-response.ts"; + +/** + * The site's canonical url, as documents in the repository spell it. Links + * that start with it are rewritten to the site serving them, the same way + * `llms.txt` builds its links. + */ +const CANONICAL_SITE_URL = "https://frontside.com/effection"; + +/** + * Matches the canonical url, but not a url that merely starts with it, so + * that `https://frontside.com/effectionx` is left alone. + */ +const CANONICAL_LINK = new RegExp( + `${CANONICAL_SITE_URL.replaceAll(".", "\\.")}(?=[/#?)\\s]|$)`, + "g", +); + +/** + * Serve the Effection behavioral contract that `llms.txt` sends agents to. + * + * `docs/agents.md` is the only copy: the file is read from the checkout on + * each request rather than duplicated here, and the root `AGENTS.md` points at + * that same file for anyone working in the repository. + * + * Its links to the documentation and the API reference are written as + * canonical urls, so a preview or a dev server rewrites them to itself and an + * agent reading them stays on the site it came from. Urls elsewhere, such as + * the Frontside blog, are left as they are. + */ +export function agentsMdRoute(): SitemapRoute { + // `.pathname` would yield `/C:/…` on Windows; `fromFileUrl` gives real paths. + let path = fromFileUrl(import.meta.resolve("../../docs/agents.md")); + + return { + *routemap(generate) { + return [{ pathname: generate() }]; + }, + *handler(): Operation { + let site = yield* url("/"); + let source = yield* until(Deno.readTextFile(path)); + + return markdown(rewriteSiteLinks(source, site)); + }, + }; +} + +export function rewriteSiteLinks(content: string, site: string): string { + // the site url ends in the slash that each canonical link already carries + return content.replaceAll(CANONICAL_LINK, site.replace(/\/$/, "")); +} diff --git a/www/routes/api-markdown-route.ts b/www/routes/api-markdown-route.ts new file mode 100644 index 000000000..a841efd6c --- /dev/null +++ b/www/routes/api-markdown-route.ts @@ -0,0 +1,142 @@ +import { type Operation } from "effection"; +import { useParams } from "revolution"; +import { toText } from "hast-util-to-text"; +import type { Nodes } from "hast"; + +import { Type } from "../components/type/jsx.tsx"; +import { useConfig } from "../context/config.ts"; +import type { DocPage, LocalDocPage } from "../hooks/use-deno-doc.tsx"; +import { createJsDocSanitizer } from "../hooks/use-markdown.tsx"; +import { + apiIndexMarkdown, + type ApiSection, + apiSymbolMarkdown, + apiSymbolPath, + type ApiVersion, +} from "../lib/api-markdown.ts"; +import { markdown, notFound } from "../lib/markdown-response.ts"; +import { usePackage } from "../lib/package.ts"; +import { url } from "../context/url.ts"; +import type { RoutePath, SitemapRoute } from "../plugins/sitemap.ts"; + +/** + * Markdown index of the API reference. + * + * Lists what the HTML index at `/api` lists — every stable series, newest + * first, with its symbols — and links to each symbol's markdown page rather + * than its page. `llms.txt` points here, so that the catalog of symbols lives + * in one place instead of being copied into it. + */ +export function apiIndexMarkdownRoute(): SitemapRoute { + return { + *routemap(generate) { + return [{ pathname: generate() }]; + }, + *handler(): Operation { + return markdown(yield* apiIndexMarkdown(yield* apiVersions())); + }, + }; +} + +/** + * Markdown page for one API symbol, from the same `pkg.docs()` the HTML page + * renders: the declaration as the page shows it, the symbol's documentation, + * and where the code lives. + */ +export function apiSymbolMarkdownRoute( + series: string, + { entrypoint = "." }: { entrypoint?: string } = {}, +): SitemapRoute { + return { + *routemap(generate): Operation { + let pages = yield* symbolPages(series, entrypoint); + + return pages.map((page) => ({ + pathname: generate({ symbol: page.name }), + })); + }, + *handler(): Operation { + let { symbol } = yield* useParams<{ symbol: string }>(); + + let pages = yield* symbolPages(series, entrypoint); + let page = pages.find((candidate) => candidate.name === symbol); + + if (!page) { + return notFound(`there is no ${series} api symbol called '${symbol}'`); + } + + let sanitize = createJsDocSanitizer(function* (name, connector, method) { + let target = pages.find((candidate) => candidate.name === name); + + if (!target) { + return [name, connector, method].filter(Boolean).join(""); + } + + let link = yield* url(apiSymbolPath(series, target)); + + return `[${ + [name, connector, method].filter(Boolean).join("") + }](${link})`; + }); + + let sections: ApiSection[] = []; + + for (let section of page.sections) { + if (!section.markdown) { + continue; + } + + sections.push({ + signature: toText( + (yield* Type({ + declaration: section.declaration, + symbol: { name: page.name }, + })) as Nodes, + ), + markdown: yield* sanitize(section.markdown), + source: section.declaration.location?.url?.toString(), + }); + } + + return markdown(apiSymbolMarkdown(page.name, sections)); + }, + }; +} + +/** + * Every stable series, newest first, the way the HTML index orders them. + */ +function* apiVersions(): Operation { + let { series } = yield* useConfig(); + let versions: ApiVersion[] = []; + + for (let entry of series.filter((s) => !s.includePrerelease).reverse()) { + let pkg = yield* usePackage({ type: "worktree", series: entry.name }); + let docs = yield* pkg.docs(); + + versions.push({ + series: entry.name, + version: pkg.version, + symbols: [ + ...(docs["."] ?? []).map(symbolOf), + ...(docs["./experimental"] ?? []).map(symbolOf), + ], + }); + } + + return versions; +} + +function symbolOf(page: DocPage) { + return { name: page.name, experimental: page.experimental }; +} + +function* symbolPages( + series: string, + entrypoint: string, +): Operation { + let pkg = yield* usePackage({ type: "worktree", series }); + let docs = yield* pkg.docs(); + + return docs[entrypoint] ?? []; +} diff --git a/www/routes/guides-markdown-route.ts b/www/routes/guides-markdown-route.ts new file mode 100644 index 000000000..7b987176a --- /dev/null +++ b/www/routes/guides-markdown-route.ts @@ -0,0 +1,57 @@ +import { all, type Operation } from "effection"; +import { useParams } from "revolution"; + +import { useConfig } from "../context/config.ts"; +import { useGuides } from "../resources/guides.ts"; +import { markdown, notFound } from "../lib/markdown-response.ts"; +import type { RoutePath, SitemapRoute } from "../plugins/sitemap.ts"; + +/** + * Serve the markdown source of a guide. + * + * `llms.txt` sends agents to the guides, and an agent wants what the guide is + * written in rather than the page it is rendered into. Serving the source from + * the site keeps a dev server or a preview from sending them to GitHub for a + * copy of the docs that belongs to a different version of the site. + */ +export function guidesMarkdownRoute(): SitemapRoute { + return { + *routemap(generate): Operation { + let { series } = yield* useConfig(); + // guides only exist for stable series, the same ones the pages cover + let stable = series.filter((s) => !s.includePrerelease); + + let paths = stable.map(function* (s) { + let pages = yield* useGuides(s.name); + + return (yield* pages.all()).map((page) => ({ + pathname: generate({ id: page.id, series: s.name }), + })); + }); + + return (yield* all(paths)).flat(); + }, + *handler(): Operation { + let { series: allSeries, current } = yield* useConfig(); + let stable = allSeries.filter((s) => !s.includePrerelease); + + let { id, series = current } = yield* useParams<{ + id: string; + series: string | undefined; + }>(); + + if (!stable.some((s) => s.name === series)) { + return notFound(`there are no guides for '${series}'`); + } + + let pages = yield* useGuides(series); + let page = yield* pages.get(id); + + if (!page) { + return notFound(`there is no guide called '${id}' in ${series}`); + } + + return markdown(page.markdown); + }, + }; +} diff --git a/www/routes/llms-txt-route.ts b/www/routes/llms-txt-route.ts index a9c2a10c8..22e50ddde 100644 --- a/www/routes/llms-txt-route.ts +++ b/www/routes/llms-txt-route.ts @@ -3,6 +3,7 @@ import { all } from "effection"; import { useWorkspaces } from "../lib/workspaces/mod.ts"; import type { SitemapRoute } from "../plugins/sitemap.ts"; import { url } from "../context/url.ts"; +import { useConfig } from "../context/config.ts"; import type { Package } from "../lib/package/types.ts"; import { groupPackagesByCategory, @@ -25,6 +26,7 @@ export function llmsTxtRoute(): SitemapRoute { return [{ pathname: generate() }]; }, *handler(): Operation { + let { current } = yield* useConfig(); let workspaces = yield* useWorkspaces("thefrontside/effectionx"); let categories = yield* useTaxonomy("thefrontside/effectionx"); let packages = yield* workspaces.getAllPackages(); @@ -55,7 +57,7 @@ export function llmsTxtRoute(): SitemapRoute { let packageLines = yield* all( category.packages.map(function* (pkg) { let shortDesc = truncateToFirstSentence(pkg.description, 120); - let href = yield* url(`/x/${pkg.workspaceName}`); + let href = yield* url(`/x/${pkg.workspaceName}.md`); return `- [${pkg.name}](${href}): ${shortDesc}`; }), @@ -80,7 +82,7 @@ export function llmsTxtRoute(): SitemapRoute { "", ...categorizedContent, "", - yield* llmsTxtFooter(), + yield* llmsTxtFooter(current), ].join("\n"); return new Response(content, { @@ -148,19 +150,34 @@ If any other document conflicts with AGENTS.md, **AGENTS.md takes precedence**. - [Resources] - [Spawn] - [Collections] - - [Browse all guides][docs/] + - [Browse all guides][Guides] --- `; -function* llmsTxtFooter(): Operation { - let [catalog, blog, api, guides] = yield* all([ +const GUIDES = [ + ["Thinking in Effection", "thinking-in-effection"], + ["Async Rosetta Stone", "async-rosetta-stone"], + ["Operations", "operations"], + ["Scope", "scope"], + ["Resources", "resources"], + ["Spawn", "spawn"], + ["Collections", "collections"], +] as const; + +function* llmsTxtFooter(series: string): Operation { + let [catalog, blog, agents, api, guides] = yield* all([ url("/x/"), url("/blog"), - url("/api/"), - url("/guides/v4"), + url("/AGENTS.md"), + url("/api.md"), + url(`/guides/${series}`), ]); + let definitions = yield* all(GUIDES.map(function* ([label, slug]) { + return `[${label}]: ${yield* url(`/guides/${series}/${slug}.md`)}`; + })); + return `## Optional - [Full EffectionX catalog with documentation](${catalog}) @@ -168,15 +185,9 @@ function* llmsTxtFooter(): Operation { --- -[AGENTS.md]: https://raw.githubusercontent.com/thefrontside/effection/v4/AGENTS.md +[AGENTS.md]: ${agents} [API]: ${api} [Guides]: ${guides} -[Thinking in Effection]: https://raw.githubusercontent.com/thefrontside/effection/v4/docs/thinking-in-effection.mdx -[Async Rosetta Stone]: https://raw.githubusercontent.com/thefrontside/effection/v4/docs/async-rosetta-stone.mdx -[Operations]: https://raw.githubusercontent.com/thefrontside/effection/v4/docs/operations.mdx -[Scope]: https://raw.githubusercontent.com/thefrontside/effection/v4/docs/scope.mdx -[Resources]: https://raw.githubusercontent.com/thefrontside/effection/v4/docs/resources.mdx -[Spawn]: https://raw.githubusercontent.com/thefrontside/effection/v4/docs/spawn.mdx -[Collections]: https://raw.githubusercontent.com/thefrontside/effection/v4/docs/collections.mdx +${definitions.join("\n")} `; } diff --git a/www/routes/x-package-markdown-route.ts b/www/routes/x-package-markdown-route.ts new file mode 100644 index 000000000..260072e1d --- /dev/null +++ b/www/routes/x-package-markdown-route.ts @@ -0,0 +1,64 @@ +import type { Operation } from "effection"; +import { useParams } from "revolution"; + +import { useWorkspaces } from "../lib/workspaces/mod.ts"; +import { markdown, notFound } from "../lib/markdown-response.ts"; +import type { RoutePath, SitemapRoute } from "../plugins/sitemap.ts"; + +/** + * Serve a package's README.md, the source of the page at `/x/:workspacePath`. + * + * An agent following the package catalog in `llms.txt` wants what the package + * says about itself, not the page it is rendered into, and it should get it + * from the site it is already reading rather than from GitHub. + */ +export function xPackageMarkdownRoute(): SitemapRoute { + return { + *routemap(generate): Operation { + let workspaces = yield* useWorkspaces("thefrontside/effectionx"); + + return (yield* workspaces.listWorkspaces()).map((workspacePath) => ({ + pathname: generate({ workspacePath }), + })); + }, + *handler(): Operation { + let { workspacePath } = yield* useParams<{ workspacePath: string }>(); + + let workspaces = yield* useWorkspaces("thefrontside/effectionx"); + let pkg = yield* workspaces.getWorkspace(workspacePath); + + if (!pkg) { + return notFound(`there is no package called '${workspacePath}'`); + } + + return markdown( + withInstallation(yield* pkg.getReadme(), yield* pkg.getName()), + ); + }, + }; +} + +/** + * Append how to install the package from npm. + * + * A readme read on its own, away from the page that carries the install + * command beside it, otherwise leaves an agent to guess the package name. The + * readmes that already give the command are left as they are, so that the + * document never says it twice. + */ +export function withInstallation(readme: string, name: string): string { + let command = `npm install ${name}`; + + if (readme.includes(command)) { + return readme; + } + + return `${readme.trimEnd()} + +## Installation + +\`\`\`sh +${command} +\`\`\` +`; +} diff --git a/www/tests/markdown-routes.ts b/www/tests/markdown-routes.ts new file mode 100644 index 000000000..baf7a943f --- /dev/null +++ b/www/tests/markdown-routes.ts @@ -0,0 +1,120 @@ +import { assertEquals, assertMatch, assertStringIncludes } from "@std/assert"; + +/** + * Smoke tests for the markdown the site serves to agents. + * + * These run against a site that is already serving, because the documents come + * from checkouts and generated API docs rather than from fixtures: + * + * deno task dev # in one terminal + * deno task smoke # in another + * + * `SMOKE_URL` points them at a different site, which is how the workflow runs + * them against the server it starts before a static build. + * + * The file is deliberately not named `*.test.ts`: `deno task test` would + * otherwise pick it up and fail, since there is no server to talk to. + */ +const SITE = Deno.env.get("SMOKE_URL") ?? "http://localhost:8000"; + +async function get(path: string): Promise<[Response, string]> { + // `SITE` is where the documents are fetched from; the urls inside them + // belong to whatever the site is configured to advertise, which in a preview + // build is not the address the tests are talking to + let response = await fetch(new URL(path, SITE)); + return [response, await response.text()]; +} + +function markdown(response: Response) { + assertEquals(response.status, 200); + assertEquals( + response.headers.get("Content-Type"), + "text/markdown; charset=utf-8", + ); +} + +Deno.test("/AGENTS.md serves the behavioral contract", async () => { + let [response, body] = await get("/AGENTS.md"); + + markdown(response); + assertStringIncludes(body, "## Core invariants (do not violate)"); + assertStringIncludes(body, "## `ensure()`"); + // the repository's own rules stay in the repository + assertEquals(body.includes("## Pre-commit workflow"), false); + // and its links lead to the site rather than to github + assertMatch(body, /consult the API reference:\nhttps?:\/\/\S+\/api\//); + assertEquals(body.includes("raw.githubusercontent.com"), false); +}); + +Deno.test("/llms.txt leads to markdown, not to github", async () => { + let [response, body] = await get("/llms.txt"); + + // the urls themselves belong to whichever site is serving, so match shape + assertEquals(response.status, 200); + assertMatch(body, /^\[AGENTS\.md\]: https?:\/\/\S+\/AGENTS\.md$/m); + assertMatch(body, /^\[API\]: https?:\/\/\S+\/api\.md$/m); + assertMatch( + body, + /^\[Operations\]: https?:\/\/\S+\/guides\/v4\/operations\.md$/m, + ); + assertMatch( + body, + /^- \[@effectionx\/task-buffer\]\(https?:\/\/\S+\/x\/task-buffer\.md\)/m, + ); + assertEquals(body.includes("github.com"), false); +}); + +Deno.test("/guides/:series/:id.md serves a guide's source", async () => { + let [response, body] = await get("/guides/v4/operations.md"); + + markdown(response); + assertStringIncludes(body, "## Stateless"); +}); + +Deno.test("/x/:package.md serves a readme that says how to install it", async () => { + let [response, body] = await get("/x/task-buffer.md"); + + markdown(response); + assertStringIncludes(body, "# Task Buffer"); + // exactly once: the readmes that already say it are left alone + assertEquals(body.split("npm install @effectionx/task-buffer").length - 1, 1); +}); + +Deno.test("/api.md indexes the symbols, /api/:series/:symbol.md documents one", async () => { + let [index, list] = await get("/api.md"); + + markdown(index); + assertStringIncludes(list, "# API Reference"); + assertMatch(list, /^- \[main\]\(https?:\/\/\S+\/api\/v4\/main\.md\)$/m); + + let [symbol, page] = await get("/api/v4/main.md"); + + markdown(symbol); + assertStringIncludes(page, "# main"); + assertMatch(page, /```ts\n.*function main/); +}); + +Deno.test("an unknown symbol is not found rather than a crash", async () => { + let [response] = await get("/api/v4/does-not-exist.md"); + + assertEquals(response.status, 404); +}); + +Deno.test("the pages these documents come from still render", async () => { + for ( + let path of [ + "/api", + "/api/v4/main", + "/guides/v4/operations", + "/x/task-buffer", + ] + ) { + let [response] = await get(path); + + assertEquals(response.status, 200, `${path} responded ${response.status}`); + assertStringIncludes( + response.headers.get("Content-Type") ?? "", + "text/html", + ); + } +});