diff --git a/README.md b/README.md index 2f9c5c3..6eb875d 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,34 @@ as `llms.txt`, markdown, feeds, json, css and javascript. Assets that are not text are copied byte for byte. This means `--base` is the only place a deployment has to say where it lives. +### Published somewhere other than where it is hosted + +A site is not always read at the address it is served from. Effection is hosted +at `effection.netlify.app` and published at `frontside.com/effection`, which is +two answers to what had been one question: where the bytes are, and what the +content is called. + +`--canonical` separates them. Urls that _name the page_ — +``, its `` twin, and +`` — are rebased onto it, while navigation, assets and the +sitemap stay on `--base`: + +``` +$ staticalize --site http://localhost:8000 --output dist \ + --base=https://interactors.netlify.app \ + --canonical=https://frontside.com/interactors +``` + +The build still browses on its own at `--base`, and search engines are told to +index `--canonical` instead. That is what a preview wants too: readable at its +own alias, indexed as production. + +Textual bodies follow `--canonical`, because a document like `llms.txt` is a map +telling a reader where the docs live rather than a page of the site. + +`--canonical` defaults to `--base`, so a site hosted where it is published says +so by saying nothing. + ### CLI ``` @@ -43,6 +71,7 @@ Arguments: Options: --output Directory to place the downloaded site [default: dist] --base Base URL of the public website. E.g. http://frontside.com + --canonical [CANONICAL] Base URL the site is published at, when that differs from where it is hosted. Only canonical urls use it. Defaults to --base. --strict Fail on the first download error instead of collecting all failures and continuing [default: false] -h, --help show help -v, --version show version diff --git a/config.ts b/config.ts index ef5e2cc..22f545c 100644 --- a/config.ts +++ b/config.ts @@ -24,6 +24,11 @@ export const config = program({ description: "Base URL of the public website. E.g. http://frontside.com", ...field(url()), }, + canonical: { + description: + "Base URL the site is published at, when that differs from where it is hosted. Only canonical urls use it. Defaults to --base.", + ...field(url().optional()), + }, strict: { description: "Fail on the first download error instead of collecting all failures and continuing", diff --git a/downloader.ts b/downloader.ts index 4cfa3ba..f76fdb9 100644 --- a/downloader.ts +++ b/downloader.ts @@ -21,6 +21,11 @@ export interface Downloader extends Operation { export interface DownloaderOptions { host: URL; base: URL; + /** + * Base url the site is published at, when that differs from where it is + * hosted. Only urls that name the page use it. Defaults to `base`. + */ + canonical?: URL; outdir: string; strict?: boolean; concurrency?: number; @@ -38,7 +43,7 @@ export const DownloadApi = createApi("@staticalize/download", { source: URL, referrer: URL, ): Operation { - let { host, base, outdir, strict, retries = 3 } = opts; + let { host, base, canonical = base, outdir, strict, retries = 3 } = opts; let signal = yield* useAbortSignal(); let path = normalize(join(outdir, source.pathname)); @@ -70,7 +75,8 @@ export const DownloadApi = createApi("@staticalize/download", { // replace self-referencing absolute urls with the destination site if (href.startsWith(host.origin)) { - link.properties.href = rebase(new URL(href), base).href; + link.properties.href = + rebase(new URL(href), names(link) ? canonical : base).href; } } @@ -103,7 +109,8 @@ export const DownloadApi = createApi("@staticalize/download", { let attr = String(element.properties.content); if (attr.startsWith(host.origin)) { yield* downloader.download(attr, source); - element.properties.content = rebase(new URL(attr), base).href; + element.properties.content = + rebase(new URL(attr), names(element) ? canonical : base).href; } } @@ -131,7 +138,11 @@ export const DownloadApi = createApi("@staticalize/download", { path, response.body! .pipeThrough(new TextDecoderStream()) - .pipeThrough(new RebaseTextStream(host, base)) + // a text body is a document about the site rather than a page of + // it — llms.txt tells an agent where the docs live, a feed tells + // a reader where the posts are — so its urls name the published + // site the way a canonical link does + .pipeThrough(new RebaseTextStream(host, canonical)) .pipeThrough(new TextEncoderStream()) .pipeThrough(counted), ), @@ -154,6 +165,25 @@ export const DownloadApi = createApi("@staticalize/download", { }, }); +/** + * Does this element name the page, rather than point into the site? + * + * A canonical link, and the `og:url` that says the same thing in another + * vocabulary, claim where the content is *published*. That is the one address + * which does not move when a build is hosted somewhere else, so it is rebased + * onto `--canonical` while navigation and assets follow `--base`. An + * `alternate` makes the same claim on behalf of another language or format. + */ +function names(element: { properties?: Record }): boolean { + let rel = element.properties?.rel; + + if (Array.isArray(rel)) { + return rel.includes("canonical") || rel.includes("alternate"); + } + + return element.properties?.property === "og:url"; +} + /** * Is this a content type whose body is text we can rewrite urls in? * diff --git a/main.ts b/main.ts index 6bacde3..f5f4715 100644 --- a/main.ts +++ b/main.ts @@ -18,8 +18,15 @@ await main(function* (args) { case "main": { let result = parser.parse(); if (result.ok) { - let { base, site, output, strict, concurrency, retries: retriesRaw } = - result.value; + let { + base, + canonical, + site, + output, + strict, + concurrency, + retries: retriesRaw, + } = result.value; // don't have a great way to default dynamically based on strict mode let retries = retriesRaw ?? (strict ? 0 : 3); @@ -33,6 +40,8 @@ await main(function* (args) { let staticalizer = yield* useStaticalizer({ base: new URL(base), + // a site hosted where it is published says so by saying nothing + canonical: canonical ? new URL(canonical) : undefined, host: new URL(site), dir: output, strict, diff --git a/staticalize.ts b/staticalize.ts index c3f2fcc..a10126f 100644 --- a/staticalize.ts +++ b/staticalize.ts @@ -15,6 +15,11 @@ import { rebase } from "./rebase.ts"; export interface StaticalizeOptions { host: URL; base: URL; + /** + * Base url the site is published at, when that differs from where it is + * hosted. Only urls that name the page use it. Defaults to `base`. + */ + canonical?: URL; dir: string; strict?: boolean; concurrency?: number; @@ -29,7 +34,8 @@ export interface Staticalizer { export function useStaticalizer( options: StaticalizeOptions, ): Operation { - let { host, base, dir, strict, concurrency, retries } = options; + let { host, base, canonical = base, dir, strict, concurrency, retries } = + options; return resource(function* (provide) { let signal = yield* useAbortSignal(); @@ -75,6 +81,7 @@ export function useStaticalizer( let downloader = yield* useDownloader({ host, base, + canonical, outdir: dir, strict, concurrency, @@ -95,6 +102,8 @@ export function useStaticalizer( urlset: { "@xmlns": "http://www.sitemaps.org/schemas/sitemap/0.9", "url": [...urls].map((url) => ({ + // the sitemap maps this deployment, so it names where the + // pages are served rather than where they are published loc: { "#text": rebase(new URL(url), base) }, })), }, diff --git a/test/staticalize.test.ts b/test/staticalize.test.ts index d9c846e..f0d1e45 100644 --- a/test/staticalize.test.ts +++ b/test/staticalize.test.ts @@ -453,6 +453,123 @@ describe("staticalize", () => { await expect(Deno.readFile("test/dist/logo.png")).resolves.toEqual(png); }); + + it("sends urls that name the page to --canonical, and the rest to --base", async () => { + app.get( + "/", + (c) => + c.html(` + + + + + + + + + + About + +`), + ) + .get("/about", (c) => c.html("

About

")) + .get("/card.png", (c) => c.text("")) + .get("/styles.css", (c) => c.text("body {}")) + .get("/main.js", (c) => c.text("console.log('hi')")) + .get(...sitemap(["/", "/about"])); + + await staticalize({ + base: new URL("https://interactors.netlify.app"), + canonical: new URL("https://frontside.com/interactors"), + host, + dir: "test/dist", + }); + + let index = await content("test/dist/index.html"); + + // these name where the page is published + expect(index).toContain( + ``, + ); + expect(index).toContain( + ``, + ); + expect(index).toContain( + ``, + ); + + // these point at bytes, which live where the build is hosted + expect(index).toContain( + ``, + ); + expect(index).toContain( + ` + + + +`), + ) + .get("/main.js", (c) => c.text("console.log('hi')")) + .get(...sitemap(["/"])); + + await staticalize({ + base: new URL("https://fs.com"), + host, + dir: "test/dist", + }); + + let index = await content("test/dist/index.html"); + expect(index).toContain(``); + expect(index).toContain(`