From d1f54e7ae2e68fc4b122af7504ecad4f37125aad Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Sat, 26 Sep 2026 08:25:33 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20honor=20the=20path=20component?= =?UTF-8?q?=20of=20--base?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both rewrite sites copied only the origin from the base, so a site published under a subpath could not be staticalized: `--base https://frontside.com/effection` produced `https://frontside.com/llms.txt`. The origin copy is now a `rebase` function that maps a url on the crawled site onto the base with the base's path as prefix, shared by the html attribute rewrite and the sitemap writer, which had the same three lines. Closes #12 --- deno.json | 1 + downloader.ts | 19 ++++------------- rebase.ts | 22 +++++++++++++++++++ staticalize.ts | 11 ++++------ test/staticalize.test.ts | 46 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 77 insertions(+), 22 deletions(-) create mode 100644 rebase.ts diff --git a/deno.json b/deno.json index 897957c..0d43dc5 100644 --- a/deno.json +++ b/deno.json @@ -45,6 +45,7 @@ "main.ts", "staticalize.ts", "downloader.ts", + "rebase.ts", "task-buffer.ts", "config.ts", "progress.ts", diff --git a/downloader.ts b/downloader.ts index 550e375..b0c08d7 100644 --- a/downloader.ts +++ b/downloader.ts @@ -11,6 +11,7 @@ import { fromHtml } from "hast-util-from-html"; import { toHtml } from "hast-util-to-html"; import { selectAll } from "hast-util-select"; import { useTaskBuffer } from "./task-buffer.ts"; +import { rebase } from "./rebase.ts"; import { createApi } from "@effectionx/context-api"; export interface Downloader extends Operation { @@ -69,11 +70,7 @@ export const DownloadApi = createApi("@staticalize/download", { // replace self-referencing absolute urls with the destination site if (href.startsWith(host.origin)) { - let url = new URL(href); - url.host = base.host; - url.port = base.port; - url.protocol = base.protocol; - link.properties.href = url.href; + link.properties.href = rebase(new URL(href), base).href; } } @@ -85,11 +82,7 @@ export const DownloadApi = createApi("@staticalize/download", { // replace self-referencing absolute urls with the destination site if (src.startsWith(host.origin)) { - let url = new URL(src); - url.host = base.host; - url.port = base.port; - url.protocol = base.protocol; - element.properties.src = url.href; + element.properties.src = rebase(new URL(src), base).href; } } @@ -98,11 +91,7 @@ export const DownloadApi = createApi("@staticalize/download", { let attr = String(element.properties.content); if (attr.startsWith(host.origin)) { yield* downloader.download(attr, source); - let url = new URL(attr); - url.host = base.host; - url.port = base.port; - url.protocol = base.protocol; - element.properties.content = url.href; + element.properties.content = rebase(new URL(attr), base).href; } } diff --git a/rebase.ts b/rebase.ts new file mode 100644 index 0000000..b12fb20 --- /dev/null +++ b/rebase.ts @@ -0,0 +1,22 @@ +/** + * Map a url on the site being crawled onto the public base url. + * + * Everything the base says about where the site lives is honored, including + * its path, so a site published under a subpath such as + * `https://frontside.com/effection` gets that prefix on every url rewritten + * into it. The source's own path, query and fragment are preserved. + * + * @param source url on the crawled site + * @param base public base url of the site, path included + * @returns the equivalent url on the base + */ +export function rebase(source: URL, base: URL): URL { + let url = new URL(source.href); + url.protocol = base.protocol; + url.host = base.host; + url.port = base.port; + // `source.pathname` always begins with a `/`, so the prefix must not end + // with one. A base without a path contributes `/`, and therefore nothing. + url.pathname = `${base.pathname.replace(/\/$/, "")}${source.pathname}`; + return url; +} diff --git a/staticalize.ts b/staticalize.ts index aa13758..c3f2fcc 100644 --- a/staticalize.ts +++ b/staticalize.ts @@ -10,6 +10,7 @@ import { ensureDir } from "@std/fs/ensure-dir"; import { stringify } from "@libs/xml/stringify"; import { parse } from "@libs/xml/parse"; import { useDownloader } from "./downloader.ts"; +import { rebase } from "./rebase.ts"; export interface StaticalizeOptions { host: URL; @@ -93,13 +94,9 @@ export function useStaticalizer( let xml = stringify({ urlset: { "@xmlns": "http://www.sitemaps.org/schemas/sitemap/0.9", - "url": [...urls].map((url) => { - let loc = new URL(url); - loc.host = base.host; - loc.port = base.port; - loc.protocol = base.protocol; - return { loc: { "#text": loc } }; - }), + "url": [...urls].map((url) => ({ + loc: { "#text": rebase(new URL(url), base) }, + })), }, }); yield* call(() => diff --git a/test/staticalize.test.ts b/test/staticalize.test.ts index c9160f0..fed628b 100644 --- a/test/staticalize.test.ts +++ b/test/staticalize.test.ts @@ -278,6 +278,52 @@ describe("staticalize", () => { ``, ); }); + + it("carries the path of the base url into every rewritten url", async () => { + app.get( + "/", + (c) => + c.html(` + + + + + + + + +`), + ) + .get("/about", (c) => c.html("

About

")) + .get("/image.png", (c) => c.text("")) + .get("/main.js", (c) => c.text("console.log('hi')")) + .get(...sitemap(["/", "/about"])); + + await staticalize({ + base: new URL("https://frontside.com/effection"), + host, + dir: "test/dist", + }); + + let index = await content("test/dist/index.html"); + expect(index).toContain( + ``, + ); + expect(index).toContain( + `