🤖 Serve the documents agents read as markdown

.github/workflows/www.yaml

@@ -68,6 +68,13 @@ jobs:
         timeout-minutes: 5
         working-directory: ./www
 
+      - name: Smoke test
+        run: deno task smoke
+        env:
+          SMOKE_URL: http://127.0.0.1:8000
+        timeout-minutes: 5
+        working-directory: ./www
+
       - name: Download Staticalize
         run: |
           wget https://github.com/thefrontside/staticalize/releases/download/v0.3.1/staticalize-linux.tar.gz \
@@ -171,6 +178,13 @@ jobs:
         timeout-minutes: 5
         working-directory: ./www
 
+      - name: Smoke test
+        run: deno task smoke
+        env:
+          SMOKE_URL: http://127.0.0.1:8000
+        timeout-minutes: 5
+        working-directory: ./www
+
       - name: Download Staticalize
         run: |
           wget https://github.com/thefrontside/staticalize/releases/download/v0.3.1/staticalize-linux.tar.gz \

AGENTS.md

@@ -5,10 +5,12 @@ repository-only rules on top of the public behavioral contract, which it does
 not repeat.
 
 Before you modify or reason about Effection code, read the public contract in
-[`docs/agents.md`](docs/agents.md), the copy checked out on this branch. It
-holds the invariants: operations versus promises, scope ownership, tasks and
-halting, context, the concurrency operations, promise interoperability,
-resources and cleanup, `ensure()`, `useAbortSignal()`, and streams.
+[`docs/agents.md`](docs/agents.md), the copy checked out on this branch. The
+website publishes the same document at
+<https://frontside.com/effection/AGENTS.md>. It holds the invariants: operations
+versus promises, scope ownership, tasks and halting, context, the concurrency
+operations, promise interoperability, resources and cleanup, `ensure()`,
+`useAbortSignal()`, and streams.
 
 Instructions scoped to a subdirectory, such as [`www/AGENTS.md`](www/AGENTS.md)
 for the website, apply in addition to this file.

www/components/footer.tsx

@@ -38,10 +38,7 @@ export function Footer(): JSX.Element {
         <a href="/llms.txt" class="text-gray-800 dark:text-gray-200">
           llms.txt
         </a>
-        <a
-          href="https://raw.githubusercontent.com/thefrontside/effection/v4/AGENTS.md"
-          class="text-gray-800 dark:text-gray-200"
-        >
+        <a href="/AGENTS.md" class="text-gray-800 dark:text-gray-200">
           AGENTS.md
         </a>
       </section>

www/deno.json

@@ -2,7 +2,8 @@
   "tasks": {
     "dev": "deno run -A @effectionx/watch deno run -A main.tsx",
     "staticalize": "deno run -A npm:staticalize@0.3.1 --site http://localhost:8000 --output=built --base=http://localhost:8000",
-    "test": "deno test --allow-run --allow-write --allow-read --allow-env"
+    "test": "deno test --allow-run --allow-write --allow-read --allow-env",
+    "smoke": "deno test --allow-net --allow-env tests/markdown-routes.ts"
   },
   "lint": {
     "exclude": [
@@ -56,6 +57,7 @@
     "hast-util-from-html": "npm:hast-util-from-html@2.0.3",
     "hast-util-shift-heading": "npm:hast-util-shift-heading@4.0.0",
     "hast-util-to-html": "npm:hast-util-to-html@9.0.0",
+    "hast-util-to-text": "npm:hast-util-to-text@4.0.2",
     "mdast": "npm:mdast@^3.0.0",
     "mdx": "npm:mdx@^0.3.1",
     "octokit": "npm:octokit@4.0.3",

www/lib/api-markdown.ts

@@ -0,0 +1,82 @@
+/**
+ * The markdown twin of the API reference: the same symbols the HTML index and
+ * symbol pages show, written out as markdown so that an agent following
+ * `llms.txt` never has to read a rendered page.
+ *
+ * These are the string builders. The routes in `routes/api-markdown-route.ts`
+ * supply the content, which comes from the same `pkg.docs()` the HTML routes
+ * render.
+ */
+
+import { all, type Operation } from "effection";
+
+import { url } from "../context/url.ts";
+
+export interface ApiSymbol {
+  name: string;
+  /** exported from the package's `./experimental` entrypoint */
+  experimental?: boolean;
+}
+
+export interface ApiVersion {
+  /** series the symbols belong to, e.g. `v4` */
+  series: string;
+  /** version of the release that series resolves to, e.g. `4.1.1` */
+  version: string;
+  symbols: ApiSymbol[];
+}
+
+export interface ApiSection {
+  /** the declaration, as the symbol page shows it */
+  signature: string;
+  /** the symbol's documentation */
+  markdown: string;
+  /** where the declaration lives */
+  source?: string;
+}
+
+/**
+ * Path of a symbol's markdown page. Experimental symbols live under an
+ * `/experimental` segment, the same as their HTML pages.
+ */
+export function apiSymbolPath(series: string, symbol: ApiSymbol): string {
+  let namespace = symbol.experimental ? `${series}/experimental` : series;
+
+  return `/api/${namespace}/${symbol.name}.md`;
+}
+
+export function* apiIndexMarkdown(
+  versions: ApiVersion[],
+): Operation<string> {
+  let sections = yield* all(
+    versions.map(function* ({ series, version, symbols }) {
+      let entries = yield* all(symbols.map(function* (symbol) {
+        let href = yield* url(apiSymbolPath(series, symbol));
+        let suffix = symbol.experimental ? " (experimental)" : "";
+
+        return `- [${symbol.name}](${href})${suffix}`;
+      }));
+
+      return [`## ${version}`, "", ...entries].join("\n");
+    }),
+  );
+
+  return `${["# API Reference", ...sections].join("\n\n")}\n`;
+}
+
+export function apiSymbolMarkdown(
+  name: string,
+  sections: ApiSection[],
+): string {
+  let bodies = sections.map(({ signature, markdown, source }) => {
+    let body = ["```ts", signature, "```", "", markdown.trim()];
+
+    if (source) {
+      body.push("", `[View code](${source})`);
+    }
+
+    return body.join("\n");
+  });
+
+  return `${[`# ${name}`, ...bodies].join("\n\n")}\n`;
+}

www/lib/markdown-response.ts

@@ -0,0 +1,28 @@
+/**
+ * Responses for the markdown the site serves to agents: the behavioral
+ * contract, the guides, and package readmes.
+ *
+ * `no-cache` rather than a max age because the header only ever reaches a
+ * browser talking to the dev server — a static build copies the body and
+ * netlify supplies its own — and these documents are rewritten per
+ * environment, so a browser must not hold one environment's copy and show it
+ * in another. The etag plugin answers the revalidation with a 304.
+ */
+export function markdown(content: string): Response {
+  return new Response(content, {
+    headers: {
+      "Content-Type": "text/markdown; charset=utf-8",
+      "Cache-Control": "no-cache",
+    },
+  });
+}
+
+export function notFound(message: string): Response {
+  return new Response(`${message}\n`, {
+    status: 404,
+    headers: {
+      "Content-Type": "text/plain; charset=utf-8",
+      "Cache-Control": "no-cache",
+    },
+  });
+}

www/main.tsx

@@ -37,6 +37,13 @@ import { redirectIndexRoute } from "./routes/redirect-index-route.tsx";
 import { searchRoute } from "./routes/search-route.tsx";
 import { initClones } from "./lib/clones.ts";
 import { initOctokitContext } from "./lib/octokit.ts";
+import { guidesMarkdownRoute } from "./routes/guides-markdown-route.ts";
+import { xPackageMarkdownRoute } from "./routes/x-package-markdown-route.ts";
+import {
+  apiIndexMarkdownRoute,
+  apiSymbolMarkdownRoute,
+} from "./routes/api-markdown-route.ts";
+import { agentsMdRoute } from "./routes/agents-md-route.ts";
 import { currentRequestPlugin } from "./plugins/current-request.ts";
 import { verboseLogging } from "./context/logging.ts";
 
@@ -97,12 +104,27 @@ function* serve(options: Options) {
       ...stableSeries.map((s) =>
         route(`/guides/${s.name}`, redirectIndexRoute(firstPage(s.name)))
       ),
+      // before the page route, so that `.md` is a suffix and not a guide id
+      route("/guides/:series/:id.md", guidesMarkdownRoute()),
       route("/guides/:series/:id", guidesRoute({ search: true })),
       route("/contrib", xIndexRedirect()),
       route("/contrib/:workspacePath", xPackageRedirect()),
       route("/x", xIndexRoute({ search: true })),
+      // before the page route, so that `.md` is a suffix and not a package
+      route("/x/:workspacePath.md", xPackageMarkdownRoute()),
       route("/x/:workspacePath", xPackageRoute({ search: true })),
       route("/api", apiIndexRoute({ search: true })),
+      // before the page routes, so that `.md` is a suffix and not a symbol
+      route("/api.md", apiIndexMarkdownRoute()),
+      ...series.map((s) =>
+        route(`/api/${s.name}/:symbol.md`, apiSymbolMarkdownRoute(s.name))
+      ),
+      ...series.map((s) =>
+        route(
+          `/api/${s.name}/experimental/:symbol.md`,
+          apiSymbolMarkdownRoute(s.name, { entrypoint: "./experimental" }),
+        )
+      ),
       // API docs for all series including prereleases
       ...series.map((s) =>
         route(
@@ -124,6 +146,7 @@ function* serve(options: Options) {
       route("/blog", blogIndexRoute({ search: true })),
       route("/blog/feed.xml", blogFeedRoute()),
       route("/llms.txt", llmsTxtRoute()),
+      route("/AGENTS.md", agentsMdRoute()),
       route("/blog/tags/:tag", blogTagRoute({ search: true })),
       route("/blog/:id", blogPostRoute({ search: true })),
       route("/blog/:id/:name.png", blogImageRoute()),

www/resources/guides.ts

@@ -1,4 +1,4 @@
-import { basename } from "@std/path";
+import { basename, toFileUrl } from "@std/path";
 import {
   all,
   createContext,
@@ -98,7 +98,11 @@ export function loadGuides(dirpath: string): Operation<Guides> {
     let loaders = new Map<string, Task<GuidesPage>>();
 
     let structureModule = yield* until(
-      import(`${dirpath}/docs/structure.json`, { with: { type: "json" } }),
+      // a path is not a module specifier on windows, where it starts with a
+      // drive letter that deno reads as an unsupported scheme
+      import(toFileUrl(`${dirpath}/docs/structure.json`).href, {
+        with: { type: "json" },
+      }),
     );
 
     let structure = Structure.parse(structureModule.default);

www/routes/agents-md-route.ts

@@ -0,0 +1,57 @@
+import type { Operation } from "effection";
+import { until } from "effection";
+import { fromFileUrl } from "@std/path";
+
+import type { SitemapRoute } from "../plugins/sitemap.ts";
+import { url } from "../context/url.ts";
+import { markdown } from "../lib/markdown-response.ts";
+
+/**
+ * The site's canonical url, as documents in the repository spell it. Links
+ * that start with it are rewritten to the site serving them, the same way
+ * `llms.txt` builds its links.
+ */
+const CANONICAL_SITE_URL = "https://frontside.com/effection";
+
+/**
+ * Matches the canonical url, but not a url that merely starts with it, so
+ * that `https://frontside.com/effectionx` is left alone.
+ */
+const CANONICAL_LINK = new RegExp(
+  `${CANONICAL_SITE_URL.replaceAll(".", "\\.")}(?=[/#?)\\s]|$)`,
+  "g",
+);
+
+/**
+ * Serve the Effection behavioral contract that `llms.txt` sends agents to.
+ *
+ * `docs/agents.md` is the only copy: the file is read from the checkout on
+ * each request rather than duplicated here, and the root `AGENTS.md` points at
+ * that same file for anyone working in the repository.
+ *
+ * Its links to the documentation and the API reference are written as
+ * canonical urls, so a preview or a dev server rewrites them to itself and an
+ * agent reading them stays on the site it came from. Urls elsewhere, such as
+ * the Frontside blog, are left as they are.
+ */
+export function agentsMdRoute(): SitemapRoute<Response> {
+  // `.pathname` would yield `/C:/…` on Windows; `fromFileUrl` gives real paths.
+  let path = fromFileUrl(import.meta.resolve("../../docs/agents.md"));
+
+  return {
+    *routemap(generate) {
+      return [{ pathname: generate() }];
+    },
+    *handler(): Operation<Response> {
+      let site = yield* url("/");
+      let source = yield* until(Deno.readTextFile(path));
+
+      return markdown(rewriteSiteLinks(source, site));
+    },
+  };
+}
+
+export function rewriteSiteLinks(content: string, site: string): string {
+  // the site url ends in the slash that each canonical link already carries
+  return content.replaceAll(CANONICAL_LINK, site.replace(/\/$/, ""));
+}