From 37140ea03f2a96ae6096accfb372178a64497d8f Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sat, 26 Sep 2026 07:42:49 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Point=20llms.txt=20and=20feed.xm?= =?UTF-8?q?l=20at=20the=20site=20they=20are=20served=20from?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both hardcoded https://frontside.com/effection, so every copy advertised production no matter where it was served from: locally llms.txt sent agents away from the dev server they were reading it on, and a preview sent reviewers to production. Everything else on the site escapes this because staticalize rewrites self referencing absolute urls — but only inside HTML. text/plain and application/rss+xml are copied to disk untouched, so their urls have to be right when the response is generated. Add `useSiteUrl()`, which builds urls from `SITE_URL` when it is set and from the origin of the request otherwise, and use it for both. The workflow sets `SITE_URL` per deploy: production advertises the canonical url, a preview advertises its own alias, and a fork, whose url is not known until after the deploy, falls back to the netlify site. The dev task sets it to localhost so development exercises the same code path. --- .github/workflows/www.yaml | 36 +++++++++++++++++++++- www/deno.json | 4 +-- www/plugins/current-request.test.ts | 47 +++++++++++++++++++++++++++++ www/plugins/current-request.ts | 27 +++++++++++++++++ www/routes/blog-feed-route.tsx | 10 +++--- www/routes/llms-txt-route.ts | 20 +++++++----- 6 files changed, 129 insertions(+), 15 deletions(-) create mode 100644 www/plugins/current-request.test.ts diff --git a/.github/workflows/www.yaml b/.github/workflows/www.yaml index ee93cf00e..303d5a020 100644 --- a/.github/workflows/www.yaml +++ b/.github/workflows/www.yaml @@ -10,6 +10,10 @@ on: permissions: contents: read +env: + # name of the netlify site, used to predict the url of a preview deploy + NETLIFY_SITE_NAME: effection + jobs: deploy-preview: if: github.event_name == 'pull_request' @@ -17,7 +21,9 @@ jobs: timeout-minutes: 15 environment: name: Preview - url: ${{ steps.netlify.outputs.unique-url }} + # the alias deploy, so that this link shares an origin with the urls + # baked into the build (see SITE_URL below) + url: ${{ steps.netlify.outputs.stable-url }} permissions: contents: read pull-requests: write @@ -33,6 +39,23 @@ jobs: with: deno-version: v2.9.1 + - name: Compute Site URL + id: site + env: + IS_FORK: ${{ github.event.pull_request.head.repo.full_name != github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + # an alias deploy lands on a predictable url, so a preview can point + # at itself. a fork deploys anonymously to a url that is not known + # until after the deploy, so it points at production instead. + if [[ "$IS_FORK" == "true" ]]; then + URL="https://${NETLIFY_SITE_NAME}.netlify.app" + else + URL="https://pr-${PR_NUMBER}--${NETLIFY_SITE_NAME}.netlify.app" + fi + echo "url=$URL" >> "$GITHUB_OUTPUT" + echo "Preview will be served from $URL" >> "$GITHUB_STEP_SUMMARY" + - name: Serve Website run: | deno run -A main.tsx & @@ -43,6 +66,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} JSR_API: ${{ secrets.JSR_API }} + # url embedded in resources that staticalize does not rewrite, such + # as llms.txt + SITE_URL: ${{ steps.site.outputs.url }} timeout-minutes: 5 working-directory: ./www @@ -90,6 +116,11 @@ jobs: DEPLOY_ID=$(echo "$DEPLOY_OUTPUT" | jq -er '.deploy_id') SITE_NAME=$(echo "$DEPLOY_OUTPUT" | jq -er '.site_name') + + if [[ "$SITE_NAME" != "$NETLIFY_SITE_NAME" ]]; then + echo "::warning::deployed to '$SITE_NAME' but urls were built for" \ + "'$NETLIFY_SITE_NAME'; update NETLIFY_SITE_NAME in this workflow" + fi UNIQUE_URL="https://${DEPLOY_ID}--${SITE_NAME}.netlify.app" STABLE_URL="https://pr-${PR_NUMBER}--${SITE_NAME}.netlify.app" fi @@ -139,6 +170,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} JSR_API: ${{ secrets.JSR_API }} + # url embedded in resources that staticalize does not rewrite, such + # as llms.txt + SITE_URL: https://frontside.com/effection timeout-minutes: 5 working-directory: ./www diff --git a/www/deno.json b/www/deno.json index 873e5351b..0a4afa658 100644 --- a/www/deno.json +++ b/www/deno.json @@ -1,8 +1,8 @@ { "tasks": { - "dev": "deno run -A @effectionx/watch deno run -A main.tsx", + "dev": "SITE_URL=http://localhost:8000 deno run -A @effectionx/watch deno run -A main.tsx", "staticalize": "deno run -A jsr:@frontside/staticalize@0.2.2/cli --site http://localhost:8000 --output=built --base=http://localhost:8000", - "test": "deno test --allow-run --allow-write --allow-read" + "test": "deno test --allow-run --allow-write --allow-read --allow-env" }, "lint": { "exclude": [ diff --git a/www/plugins/current-request.test.ts b/www/plugins/current-request.test.ts new file mode 100644 index 000000000..98444f80b --- /dev/null +++ b/www/plugins/current-request.test.ts @@ -0,0 +1,47 @@ +import { describe, it } from "../testing.ts"; +import { expect } from "expect"; + +import { CurrentRequest } from "../context/request.ts"; +import { useSiteUrl } from "./current-request.ts"; + +describe("useSiteUrl", () => { + it("uses the origin of the current request by default", function* () { + yield* CurrentRequest.set(new Request("http://localhost:8000/llms.txt")); + Deno.env.delete("SITE_URL"); + + let url = yield* useSiteUrl(); + + expect(url("/x/")).toEqual("http://localhost:8000/x/"); + expect(url("/x/websocket")).toEqual("http://localhost:8000/x/websocket"); + }); + + it("uses SITE_URL when it has no path of its own", function* () { + yield* CurrentRequest.set(new Request("http://127.0.0.1:8000/llms.txt")); + Deno.env.set("SITE_URL", "https://pr-42--effection.netlify.app"); + + try { + let url = yield* useSiteUrl(); + + expect(url("/x/")).toEqual("https://pr-42--effection.netlify.app/x/"); + expect(url("/blog")).toEqual( + "https://pr-42--effection.netlify.app/blog", + ); + } finally { + Deno.env.delete("SITE_URL"); + } + }); + + it("uses SITE_URL when it is set, preserving its path", function* () { + yield* CurrentRequest.set(new Request("http://127.0.0.1:8000/llms.txt")); + Deno.env.set("SITE_URL", "https://frontside.com/effection"); + + try { + let url = yield* useSiteUrl(); + + expect(url("/x/")).toEqual("https://frontside.com/effection/x/"); + expect(url("/blog")).toEqual("https://frontside.com/effection/blog"); + } finally { + Deno.env.delete("SITE_URL"); + } + }); +}); diff --git a/www/plugins/current-request.ts b/www/plugins/current-request.ts index 6347074f7..27876e94f 100644 --- a/www/plugins/current-request.ts +++ b/www/plugins/current-request.ts @@ -45,3 +45,30 @@ export function* useCanonicalUrl(options: { base: string }): Operation { url.pathname = `${url.pathname}${req.pathname}`; return String(url); } + +/** + * Like {@link useAbsoluteUrlFactory}, except that it honors the `SITE_URL` + * of the published site when one is configured. + * + * Absolute urls in HTML are rewritten to the destination site by staticalize + * when the site is built, so they can just use the origin of the request. + * Urls inside non HTML resources such as `llms.txt` are copied verbatim, so + * they need to be published under `SITE_URL` instead of the loopback address + * that the build serves from. With no `SITE_URL`, they point at the dev + * server, same as every other absolute url. + */ +export function* useSiteUrl(): Operation<(path: string) => string> { + let siteUrl = Deno.env.get("SITE_URL"); + + if (!siteUrl) { + return yield* useAbsoluteUrlFactory(); + } + + let base = new URL(siteUrl); + + return (path) => { + let url = new URL(base); + url.pathname = posixNormalize(`${base.pathname}/${path}`); + return url.toString(); + }; +} diff --git a/www/routes/blog-feed-route.tsx b/www/routes/blog-feed-route.tsx index 47921e54e..80ea4fec2 100644 --- a/www/routes/blog-feed-route.tsx +++ b/www/routes/blog-feed-route.tsx @@ -2,6 +2,7 @@ import type { Operation } from "effection"; import { stringify } from "@libs/xml"; import { useBlog } from "../resources/blog.ts"; +import { useSiteUrl } from "../plugins/current-request.ts"; /** * RSS 2.0 feed for the blog @@ -11,8 +12,7 @@ export function blogFeedRoute() { *handler(): Operation { let blog = yield* useBlog(); let posts = blog.getPosts(); - - let baseUrl = "https://frontside.com/effection"; + let url = yield* useSiteUrl(); let xml = stringify({ "@version": "1.0", @@ -22,18 +22,18 @@ export function blogFeedRoute() { "@xmlns:atom": "http://www.w3.org/2005/Atom", channel: { title: "Effection Blog", - link: `${baseUrl}/blog`, + link: url("/blog"), description: "Tutorials, announcements, and insights about structured concurrency in JavaScript with Effection.", language: "en-us", lastBuildDate: new Date().toUTCString(), "atom:link": { - "@href": `${baseUrl}/blog/feed.xml`, + "@href": url("/blog/feed.xml"), "@rel": "self", "@type": "application/rss+xml", }, item: posts.slice(0, 20).map((post) => { - let postUrl = `${baseUrl}/blog/${post.id}/`; + let postUrl = url(`/blog/${post.id}/`); return { title: post.title, link: postUrl, diff --git a/www/routes/llms-txt-route.ts b/www/routes/llms-txt-route.ts index dd7c1cd00..29a5c0cd6 100644 --- a/www/routes/llms-txt-route.ts +++ b/www/routes/llms-txt-route.ts @@ -2,6 +2,7 @@ import type { Operation } from "effection"; import { all } from "effection"; import { useWorkspaces } from "../lib/workspaces/mod.ts"; import type { SitemapRoute } from "../plugins/sitemap.ts"; +import { useSiteUrl } from "../plugins/current-request.ts"; import type { Package } from "../lib/package/types.ts"; import { groupPackagesByCategory, @@ -24,6 +25,7 @@ export function llmsTxtRoute(): SitemapRoute { return [{ pathname: generate() }]; }, *handler(): Operation { + let url = yield* useSiteUrl(); let workspaces = yield* useWorkspaces("thefrontside/effectionx"); let categories = yield* useTaxonomy("thefrontside/effectionx"); let packages = yield* workspaces.getAllPackages(); @@ -52,7 +54,9 @@ export function llmsTxtRoute(): SitemapRoute { (category) => { let packageLines = category.packages.map((pkg) => { let shortDesc = truncateToFirstSentence(pkg.description, 120); - return `- [${pkg.name}](https://frontside.com/effection/x/${pkg.workspaceName}): ${shortDesc}`; + return `- [${pkg.name}](${ + url(`/x/${pkg.workspaceName}`) + }): ${shortDesc}`; }); return [ @@ -73,7 +77,7 @@ export function llmsTxtRoute(): SitemapRoute { "", ...categorizedContent, "", - LLMS_TXT_FOOTER, + llmsTxtFooter(url), ].join("\n"); return new Response(content, { @@ -146,16 +150,17 @@ If any other document conflicts with AGENTS.md, **AGENTS.md takes precedence**. --- `; -const LLMS_TXT_FOOTER = `## Optional +function llmsTxtFooter(url: (path: string) => string): string { + return `## Optional -- [Full EffectionX catalog with documentation](https://frontside.com/effection/x/) -- [Effection Blog](https://frontside.com/effection/blog) +- [Full EffectionX catalog with documentation](${url("/x/")}) +- [Effection Blog](${url("/blog")}) --- [AGENTS.md]: https://raw.githubusercontent.com/thefrontside/effection/v4/AGENTS.md -[API]: https://frontside.com/effection/api/ -[Guides]: https://frontside.com/effection/guides/v4 +[API]: ${url("/api/")} +[Guides]: ${url("/guides/v4")} [Thinking in Effection]: https://raw.githubusercontent.com/thefrontside/effection/v4/docs/thinking-in-effection.mdx [Async Rosetta Stone]: https://raw.githubusercontent.com/thefrontside/effection/v4/docs/async-rosetta-stone.mdx [Operations]: https://raw.githubusercontent.com/thefrontside/effection/v4/docs/operations.mdx @@ -164,3 +169,4 @@ const LLMS_TXT_FOOTER = `## Optional [Spawn]: https://raw.githubusercontent.com/thefrontside/effection/v4/docs/spawn.mdx [Collections]: https://raw.githubusercontent.com/thefrontside/effection/v4/docs/collections.mdx `; +}