✨ Separate where a build is hosted from the url it is published at

README.md

@@ -32,6 +32,34 @@ as `llms.txt`, markdown, feeds, json, css and javascript. Assets that are not
 text are copied byte for byte. This means `--base` is the only place a
 deployment has to say where it lives.
 
+### Published somewhere other than where it is hosted
+
+A site is not always read at the address it is served from. Effection is hosted
+at `effection.netlify.app` and published at `frontside.com/effection`, which is
+two answers to what had been one question: where the bytes are, and what the
+content is called.
+
+`--canonical` separates them. Urls that _name the page_ —
+`<link rel=canonical>`, its `<meta property="og:url">` twin, and
+`<link rel=alternate>` — are rebased onto it, while navigation, assets and the
+sitemap stay on `--base`:
+
+```
+$ staticalize --site http://localhost:8000 --output dist \
+    --base=https://interactors.netlify.app \
+    --canonical=https://frontside.com/interactors
+```
+
+The build still browses on its own at `--base`, and search engines are told to
+index `--canonical` instead. That is what a preview wants too: readable at its
+own alias, indexed as production.
+
+Textual bodies follow `--canonical`, because a document like `llms.txt` is a map
+telling a reader where the docs live rather than a page of the site.
+
+`--canonical` defaults to `--base`, so a site hosted where it is published says
+so by saying nothing.
+
 ### CLI
 
 ```
@@ -43,6 +71,7 @@ Arguments:
 Options:
    --output <OUTPUT>         Directory to place the downloaded site [default: dist]
    --base <BASE>             Base URL of the public website. E.g. http://frontside.com
+   --canonical [CANONICAL]   Base URL the site is published at, when that differs from where it is hosted. Only canonical urls use it. Defaults to --base.
    --strict                  Fail on the first download error instead of collecting all failures and continuing [default: false]
    -h, --help                show help
    -v, --version             show version

config.ts

@@ -24,6 +24,11 @@ export const config = program({
       description: "Base URL of the public website. E.g. http://frontside.com",
       ...field(url()),
     },
+    canonical: {
+      description:
+        "Base URL the site is published at, when that differs from where it is hosted. Only canonical urls use it. Defaults to --base.",
+      ...field(url().optional()),
+    },
     strict: {
       description:
         "Fail on the first download error instead of collecting all failures and continuing",

downloader.ts

@@ -21,6 +21,11 @@ export interface Downloader extends Operation<void> {
 export interface DownloaderOptions {
   host: URL;
   base: URL;
+  /**
+   * Base url the site is published at, when that differs from where it is
+   * hosted. Only urls that name the page use it. Defaults to `base`.
+   */
+  canonical?: URL;
   outdir: string;
   strict?: boolean;
   concurrency?: number;
@@ -38,7 +43,7 @@ export const DownloadApi = createApi("@staticalize/download", {
     source: URL,
     referrer: URL,
   ): Operation<DownloadResult> {
-    let { host, base, outdir, strict, retries = 3 } = opts;
+    let { host, base, canonical = base, outdir, strict, retries = 3 } = opts;
     let signal = yield* useAbortSignal();
     let path = normalize(join(outdir, source.pathname));
 
@@ -70,7 +75,8 @@ export const DownloadApi = createApi("@staticalize/download", {
 
           // replace self-referencing absolute urls with the destination site
           if (href.startsWith(host.origin)) {
-            link.properties.href = rebase(new URL(href), base).href;
+            link.properties.href =
+              rebase(new URL(href), names(link) ? canonical : base).href;
           }
         }
 
@@ -103,7 +109,8 @@ export const DownloadApi = createApi("@staticalize/download", {
           let attr = String(element.properties.content);
           if (attr.startsWith(host.origin)) {
             yield* downloader.download(attr, source);
-            element.properties.content = rebase(new URL(attr), base).href;
+            element.properties.content =
+              rebase(new URL(attr), names(element) ? canonical : base).href;
           }
         }
 
@@ -131,7 +138,11 @@ export const DownloadApi = createApi("@staticalize/download", {
             path,
             response.body!
               .pipeThrough(new TextDecoderStream())
-              .pipeThrough(new RebaseTextStream(host, base))
+              // a text body is a document about the site rather than a page of
+              // it — llms.txt tells an agent where the docs live, a feed tells
+              // a reader where the posts are — so its urls name the published
+              // site the way a canonical link does
+              .pipeThrough(new RebaseTextStream(host, canonical))
               .pipeThrough(new TextEncoderStream())
               .pipeThrough(counted),
           ),
@@ -154,6 +165,25 @@ export const DownloadApi = createApi("@staticalize/download", {
   },
 });
 
+/**
+ * Does this element name the page, rather than point into the site?
+ *
+ * A canonical link, and the `og:url` that says the same thing in another
+ * vocabulary, claim where the content is *published*. That is the one address
+ * which does not move when a build is hosted somewhere else, so it is rebased
+ * onto `--canonical` while navigation and assets follow `--base`. An
+ * `alternate` makes the same claim on behalf of another language or format.
+ */
+function names(element: { properties?: Record<string, unknown> }): boolean {
+  let rel = element.properties?.rel;
+
+  if (Array.isArray(rel)) {
+    return rel.includes("canonical") || rel.includes("alternate");
+  }
+
+  return element.properties?.property === "og:url";
+}
+
 /**
  * Is this a content type whose body is text we can rewrite urls in?
  *

main.ts

@@ -18,8 +18,15 @@ await main(function* (args) {
     case "main": {
       let result = parser.parse();
       if (result.ok) {
-        let { base, site, output, strict, concurrency, retries: retriesRaw } =
-          result.value;
+        let {
+          base,
+          canonical,
+          site,
+          output,
+          strict,
+          concurrency,
+          retries: retriesRaw,
+        } = result.value;
         // don't have a great way to default dynamically based on strict mode
         let retries = retriesRaw ?? (strict ? 0 : 3);
 
@@ -33,6 +40,8 @@ await main(function* (args) {
 
         let staticalizer = yield* useStaticalizer({
           base: new URL(base),
+          // a site hosted where it is published says so by saying nothing
+          canonical: canonical ? new URL(canonical) : undefined,
           host: new URL(site),
           dir: output,
           strict,

staticalize.ts

@@ -15,6 +15,11 @@ import { rebase } from "./rebase.ts";
 export interface StaticalizeOptions {
   host: URL;
   base: URL;
+  /**
+   * Base url the site is published at, when that differs from where it is
+   * hosted. Only urls that name the page use it. Defaults to `base`.
+   */
+  canonical?: URL;
   dir: string;
   strict?: boolean;
   concurrency?: number;
@@ -29,7 +34,8 @@ export interface Staticalizer {
 export function useStaticalizer(
   options: StaticalizeOptions,
 ): Operation<Staticalizer> {
-  let { host, base, dir, strict, concurrency, retries } = options;
+  let { host, base, canonical = base, dir, strict, concurrency, retries } =
+    options;
 
   return resource(function* (provide) {
     let signal = yield* useAbortSignal();
@@ -75,6 +81,7 @@ export function useStaticalizer(
     let downloader = yield* useDownloader({
       host,
       base,
+      canonical,
       outdir: dir,
       strict,
       concurrency,
@@ -95,6 +102,8 @@ export function useStaticalizer(
             urlset: {
               "@xmlns": "http://www.sitemaps.org/schemas/sitemap/0.9",
               "url": [...urls].map((url) => ({
+                // the sitemap maps this deployment, so it names where the
+                // pages are served rather than where they are published
                 loc: { "#text": rebase(new URL(url), base) },
               })),
             },

test/staticalize.test.ts

@@ -453,6 +453,123 @@ describe("staticalize", () => {
 
     await expect(Deno.readFile("test/dist/logo.png")).resolves.toEqual(png);
   });
+
+  it("sends urls that name the page to --canonical, and the rest to --base", async () => {
+    app.get(
+      "/",
+      (c) =>
+        c.html(`
+<html>
+  <head>
+    <link rel="canonical" href="${host}"/>
+    <link rel="alternate" href="${host}" hreflang="en"/>
+    <meta property="og:url" content="${host}"/>
+    <meta property="og:image" content="${host}card.png"/>
+    <link rel="stylesheet" href="${host}styles.css"/>
+    <script src="${host}main.js"></script>
+  </head>
+  <body><a href="${host}about">About</a></body>
+</html>
+`),
+    )
+      .get("/about", (c) => c.html("<h1>About</h1>"))
+      .get("/card.png", (c) => c.text(""))
+      .get("/styles.css", (c) => c.text("body {}"))
+      .get("/main.js", (c) => c.text("console.log('hi')"))
+      .get(...sitemap(["/", "/about"]));
+
+    await staticalize({
+      base: new URL("https://interactors.netlify.app"),
+      canonical: new URL("https://frontside.com/interactors"),
+      host,
+      dir: "test/dist",
+    });
+
+    let index = await content("test/dist/index.html");
+
+    // these name where the page is published
+    expect(index).toContain(
+      `<link rel="canonical" href="https://frontside.com/interactors/">`,
+    );
+    expect(index).toContain(
+      `<link rel="alternate" href="https://frontside.com/interactors/" hreflang="en">`,
+    );
+    expect(index).toContain(
+      `<meta property="og:url" content="https://frontside.com/interactors/">`,
+    );
+
+    // these point at bytes, which live where the build is hosted
+    expect(index).toContain(
+      `<link rel="stylesheet" href="https://interactors.netlify.app/styles.css">`,
+    );
+    expect(index).toContain(
+      `<script src="https://interactors.netlify.app/main.js">`,
+    );
+    expect(index).toContain(
+      `<a href="https://interactors.netlify.app/about">`,
+    );
+    expect(index).toContain(
+      `<meta property="og:image" content="https://interactors.netlify.app/card.png">`,
+    );
+
+    // the sitemap maps this deployment, so it stays on the host
+    let sitemapXML = await Deno.readTextFile("test/dist/sitemap.xml");
+    expect(sitemapXML).toContain("https://interactors.netlify.app/about");
+    expect(sitemapXML).not.toContain("frontside.com");
+  });
+
+  it("names the published site in text bodies", async () => {
+    app.get("/", (c) => c.html("<h1>Home</h1>"))
+      .get(
+        "/llms.txt",
+        (c) =>
+          c.text(`[Docs]: ${host}docs\n[Blog]: ${host}blog`, 200, {
+            "Content-Type": "text/plain",
+          }),
+      )
+      .get(...sitemap(["/", "/llms.txt"]));
+
+    await staticalize({
+      base: new URL("https://interactors.netlify.app"),
+      canonical: new URL("https://frontside.com/interactors"),
+      host,
+      dir: "test/dist",
+    });
+
+    // llms.txt tells an agent where the docs live, which is the published site
+    let llms = await content("test/dist/llms.txt");
+    expect(llms).toContain("https://frontside.com/interactors/docs");
+    expect(llms).toContain("https://frontside.com/interactors/blog");
+    expect(llms).not.toContain("netlify");
+  });
+
+  it("leaves everything on --base when --canonical is not given", async () => {
+    app.get(
+      "/",
+      (c) =>
+        c.html(`
+<html>
+  <head>
+    <link rel="canonical" href="${host}"/>
+    <script src="${host}main.js"></script>
+  </head>
+  <body></body>
+</html>
+`),
+    )
+      .get("/main.js", (c) => c.text("console.log('hi')"))
+      .get(...sitemap(["/"]));
+
+    await staticalize({
+      base: new URL("https://fs.com"),
+      host,
+      dir: "test/dist",
+    });
+
+    let index = await content("test/dist/index.html");
+    expect(index).toContain(`<link rel="canonical" href="https://fs.com/">`);
+    expect(index).toContain(`<script src="https://fs.com/main.js">`);
+  });
 });
 
 async function content(path: string): Promise<string> {