From d9dbe60dba2d8a17af39298c4c4fc74f64460855 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Mon, 28 Sep 2026 14:05:08 -0400 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Separate=20where=20a=20build=20is?= =?UTF-8?q?=20hosted=20from=20the=20url=20it=20is=20published=20at?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--base` answered two questions at once: where these bytes live, and what this content is called. They coincide for an ordinary site and come apart the moment one is served from one origin and read at another — Effection is hosted at effection.netlify.app and published at frontside.com/effection. Getting the canonical right meant pointing `--base` at the published url, which dragged every other url along with it. thefrontside/effection#1248 did exactly that, and the sitemap went with it: frontside.com prefixes `/effection` onto a proxied sitemap, the entries already carried it, and 519 urls 404'd. `--canonical` splits the two. Urls that name the page — a canonical link, the `og:url` that says the same thing, an `alternate` saying it for another language — rebase onto it. Navigation, assets and the sitemap stay on `--base`, so the build still browses at the address it is served from. Textual bodies follow `--canonical`: llms.txt is a map telling a reader where the docs live, not a page of the site. It defaults to `--base`, so every existing invocation is unchanged, and a site hosted where it is published says so by saying nothing. Closes #21. --- README.md | 29 ++++++++++ config.ts | 5 ++ downloader.ts | 38 +++++++++++-- main.ts | 13 ++++- staticalize.ts | 11 +++- test/staticalize.test.ts | 117 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 206 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 2f9c5c3..6eb875d 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,34 @@ as `llms.txt`, markdown, feeds, json, css and javascript. Assets that are not text are copied byte for byte. This means `--base` is the only place a deployment has to say where it lives. +### Published somewhere other than where it is hosted + +A site is not always read at the address it is served from. Effection is hosted +at `effection.netlify.app` and published at `frontside.com/effection`, which is +two answers to what had been one question: where the bytes are, and what the +content is called. + +`--canonical` separates them. Urls that _name the page_ — +``, its `` twin, and +`` — are rebased onto it, while navigation, assets and the +sitemap stay on `--base`: + +``` +$ staticalize --site http://localhost:8000 --output dist \ + --base=https://interactors.netlify.app \ + --canonical=https://frontside.com/interactors +``` + +The build still browses on its own at `--base`, and search engines are told to +index `--canonical` instead. That is what a preview wants too: readable at its +own alias, indexed as production. + +Textual bodies follow `--canonical`, because a document like `llms.txt` is a map +telling a reader where the docs live rather than a page of the site. + +`--canonical` defaults to `--base`, so a site hosted where it is published says +so by saying nothing. + ### CLI ``` @@ -43,6 +71,7 @@ Arguments: Options: --output Directory to place the downloaded site [default: dist] --base Base URL of the public website. E.g. http://frontside.com + --canonical [CANONICAL] Base URL the site is published at, when that differs from where it is hosted. Only canonical urls use it. Defaults to --base. --strict Fail on the first download error instead of collecting all failures and continuing [default: false] -h, --help show help -v, --version show version diff --git a/config.ts b/config.ts index ef5e2cc..22f545c 100644 --- a/config.ts +++ b/config.ts @@ -24,6 +24,11 @@ export const config = program({ description: "Base URL of the public website. E.g. http://frontside.com", ...field(url()), }, + canonical: { + description: + "Base URL the site is published at, when that differs from where it is hosted. Only canonical urls use it. Defaults to --base.", + ...field(url().optional()), + }, strict: { description: "Fail on the first download error instead of collecting all failures and continuing", diff --git a/downloader.ts b/downloader.ts index 4cfa3ba..f76fdb9 100644 --- a/downloader.ts +++ b/downloader.ts @@ -21,6 +21,11 @@ export interface Downloader extends Operation { export interface DownloaderOptions { host: URL; base: URL; + /** + * Base url the site is published at, when that differs from where it is + * hosted. Only urls that name the page use it. Defaults to `base`. + */ + canonical?: URL; outdir: string; strict?: boolean; concurrency?: number; @@ -38,7 +43,7 @@ export const DownloadApi = createApi("@staticalize/download", { source: URL, referrer: URL, ): Operation { - let { host, base, outdir, strict, retries = 3 } = opts; + let { host, base, canonical = base, outdir, strict, retries = 3 } = opts; let signal = yield* useAbortSignal(); let path = normalize(join(outdir, source.pathname)); @@ -70,7 +75,8 @@ export const DownloadApi = createApi("@staticalize/download", { // replace self-referencing absolute urls with the destination site if (href.startsWith(host.origin)) { - link.properties.href = rebase(new URL(href), base).href; + link.properties.href = + rebase(new URL(href), names(link) ? canonical : base).href; } } @@ -103,7 +109,8 @@ export const DownloadApi = createApi("@staticalize/download", { let attr = String(element.properties.content); if (attr.startsWith(host.origin)) { yield* downloader.download(attr, source); - element.properties.content = rebase(new URL(attr), base).href; + element.properties.content = + rebase(new URL(attr), names(element) ? canonical : base).href; } } @@ -131,7 +138,11 @@ export const DownloadApi = createApi("@staticalize/download", { path, response.body! .pipeThrough(new TextDecoderStream()) - .pipeThrough(new RebaseTextStream(host, base)) + // a text body is a document about the site rather than a page of + // it — llms.txt tells an agent where the docs live, a feed tells + // a reader where the posts are — so its urls name the published + // site the way a canonical link does + .pipeThrough(new RebaseTextStream(host, canonical)) .pipeThrough(new TextEncoderStream()) .pipeThrough(counted), ), @@ -154,6 +165,25 @@ export const DownloadApi = createApi("@staticalize/download", { }, }); +/** + * Does this element name the page, rather than point into the site? + * + * A canonical link, and the `og:url` that says the same thing in another + * vocabulary, claim where the content is *published*. That is the one address + * which does not move when a build is hosted somewhere else, so it is rebased + * onto `--canonical` while navigation and assets follow `--base`. An + * `alternate` makes the same claim on behalf of another language or format. + */ +function names(element: { properties?: Record }): boolean { + let rel = element.properties?.rel; + + if (Array.isArray(rel)) { + return rel.includes("canonical") || rel.includes("alternate"); + } + + return element.properties?.property === "og:url"; +} + /** * Is this a content type whose body is text we can rewrite urls in? * diff --git a/main.ts b/main.ts index 6bacde3..f5f4715 100644 --- a/main.ts +++ b/main.ts @@ -18,8 +18,15 @@ await main(function* (args) { case "main": { let result = parser.parse(); if (result.ok) { - let { base, site, output, strict, concurrency, retries: retriesRaw } = - result.value; + let { + base, + canonical, + site, + output, + strict, + concurrency, + retries: retriesRaw, + } = result.value; // don't have a great way to default dynamically based on strict mode let retries = retriesRaw ?? (strict ? 0 : 3); @@ -33,6 +40,8 @@ await main(function* (args) { let staticalizer = yield* useStaticalizer({ base: new URL(base), + // a site hosted where it is published says so by saying nothing + canonical: canonical ? new URL(canonical) : undefined, host: new URL(site), dir: output, strict, diff --git a/staticalize.ts b/staticalize.ts index c3f2fcc..a10126f 100644 --- a/staticalize.ts +++ b/staticalize.ts @@ -15,6 +15,11 @@ import { rebase } from "./rebase.ts"; export interface StaticalizeOptions { host: URL; base: URL; + /** + * Base url the site is published at, when that differs from where it is + * hosted. Only urls that name the page use it. Defaults to `base`. + */ + canonical?: URL; dir: string; strict?: boolean; concurrency?: number; @@ -29,7 +34,8 @@ export interface Staticalizer { export function useStaticalizer( options: StaticalizeOptions, ): Operation { - let { host, base, dir, strict, concurrency, retries } = options; + let { host, base, canonical = base, dir, strict, concurrency, retries } = + options; return resource(function* (provide) { let signal = yield* useAbortSignal(); @@ -75,6 +81,7 @@ export function useStaticalizer( let downloader = yield* useDownloader({ host, base, + canonical, outdir: dir, strict, concurrency, @@ -95,6 +102,8 @@ export function useStaticalizer( urlset: { "@xmlns": "http://www.sitemaps.org/schemas/sitemap/0.9", "url": [...urls].map((url) => ({ + // the sitemap maps this deployment, so it names where the + // pages are served rather than where they are published loc: { "#text": rebase(new URL(url), base) }, })), }, diff --git a/test/staticalize.test.ts b/test/staticalize.test.ts index d9c846e..f0d1e45 100644 --- a/test/staticalize.test.ts +++ b/test/staticalize.test.ts @@ -453,6 +453,123 @@ describe("staticalize", () => { await expect(Deno.readFile("test/dist/logo.png")).resolves.toEqual(png); }); + + it("sends urls that name the page to --canonical, and the rest to --base", async () => { + app.get( + "/", + (c) => + c.html(` + + + + + + + + + + About + +`), + ) + .get("/about", (c) => c.html("

About

")) + .get("/card.png", (c) => c.text("")) + .get("/styles.css", (c) => c.text("body {}")) + .get("/main.js", (c) => c.text("console.log('hi')")) + .get(...sitemap(["/", "/about"])); + + await staticalize({ + base: new URL("https://interactors.netlify.app"), + canonical: new URL("https://frontside.com/interactors"), + host, + dir: "test/dist", + }); + + let index = await content("test/dist/index.html"); + + // these name where the page is published + expect(index).toContain( + ``, + ); + expect(index).toContain( + ``, + ); + expect(index).toContain( + ``, + ); + + // these point at bytes, which live where the build is hosted + expect(index).toContain( + ``, + ); + expect(index).toContain( + ` + + + +`), + ) + .get("/main.js", (c) => c.text("console.log('hi')")) + .get(...sitemap(["/"])); + + await staticalize({ + base: new URL("https://fs.com"), + host, + dir: "test/dist", + }); + + let index = await content("test/dist/index.html"); + expect(index).toContain(``); + expect(index).toContain(`