diff --git a/package.json b/package.json index 54d4ad48d3..e146ae56e7 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "private": true, "type": "module", "scripts": { - "docs:build": "node scripts/docs-site/build.mjs && node scripts/docs-site/search-index.mjs && node scripts/docs-site/source-index.mjs && pagefind --site dist/docs-site --output-path dist/docs-site/pagefind && node scripts/docs-site/pagefind-normalize.mjs", + "docs:build": "node scripts/docs-site/build.mjs && node scripts/docs-site/search-index.mjs && node scripts/docs-site/source-index.mjs && node scripts/docs-site/pagefind-build.mjs && node scripts/docs-site/pagefind-normalize.mjs", "docs:build:shell": "DOCS_SITE_ARTIFACT_MODE=shell node scripts/docs-site/build.mjs", "docs:build:preview": "DOCS_SITE_ARTIFACT_MODE=shell DOCS_SITE_PREVIEW_LOCALE=en DOCS_SITE_PREVIEW_PAGES_PER_GROUP=2 DOCS_SITE_PREVIEW_MAX_PAGES=30 node scripts/docs-site/build.mjs", "docs:build:r2": "npm run docs:build && node scripts/docs-site/r2-prepare.mjs", diff --git a/scripts/docs-site/analytics-behavior.test.mjs b/scripts/docs-site/analytics-behavior.test.mjs index 328dca3bc2..dd7257115e 100644 --- a/scripts/docs-site/analytics-behavior.test.mjs +++ b/scripts/docs-site/analytics-behavior.test.mjs @@ -4,6 +4,7 @@ import path from "node:path"; import { before, after, test } from "node:test"; import { chromium } from "playwright"; import { fixture, write } from "./test-helpers/redirect-fixture.mjs"; +import { pagefindAssetDir } from "./pagefind-assets.mjs"; const origin = "https://docs.openclaw.ai"; const sdk = process.env.DOCS_GA4_SDK_PATH ? fs.readFileSync(process.env.DOCS_GA4_SDK_PATH, "utf8") : ""; @@ -43,7 +44,7 @@ async function openSite(t, { privacy, clock = false, diagram = false, articleLin if (url.pathname === "/ask-molty/api/chat") return route.fulfill(request.method() === "GET" ? { contentType: "application/json", body: "{}" } : { contentType: "text/plain", body: "[Private reference](https://private.example/PRIVATE_CHAT_VALUE?q=PRIVATE_CHAT_QUERY)" }); - if (url.pathname === "/pagefind/pagefind.js") return route.fulfill({ contentType: "text/javascript", body: ` + if (url.pathname === `/${pagefindAssetDir}/pagefind.js`) return route.fulfill({ contentType: "text/javascript", body: ` export async function init() {} export async function search(q) { if(q.includes("delay")) await window.__searchWait; if(q.includes("error")) throw new Error("Fixture search error"); return {results: Array.from({length:q.includes("missing")?0:q.includes("many")?20:1},(_,i)=>({data: async () => ({url:"/guide#result-"+i,meta:{title:"Guide"},excerpt:"Public guidance"})}))}; } ` }); diff --git a/scripts/docs-site/pagefind-assets.mjs b/scripts/docs-site/pagefind-assets.mjs new file mode 100644 index 0000000000..32a1a8cbff --- /dev/null +++ b/scripts/docs-site/pagefind-assets.mjs @@ -0,0 +1,10 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const packageRoot = path.dirname(path.dirname(fileURLToPath(import.meta.resolve("pagefind")))); +export const pagefindVersion = JSON.parse(fs.readFileSync(path.join(packageRoot, "package.json"), "utf8")).version; +if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(pagefindVersion)) { + throw new Error(`Invalid installed Pagefind version: ${pagefindVersion}`); +} +export const pagefindAssetDir = `pagefind/v${pagefindVersion}`; diff --git a/scripts/docs-site/pagefind-build.mjs b/scripts/docs-site/pagefind-build.mjs new file mode 100644 index 0000000000..79e15e26ad --- /dev/null +++ b/scripts/docs-site/pagefind-build.mjs @@ -0,0 +1,16 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { pagefindAssetDir } from "./pagefind-assets.mjs"; + +const site = path.join(process.cwd(), "dist/docs-site"); +if (!fs.existsSync(site)) throw new Error("dist/docs-site does not exist; run build.mjs first"); + +const runner = fileURLToPath(new URL("../../node_modules/pagefind/lib/runner/bin.cjs", import.meta.url)); +const result = spawnSync(process.execPath, [runner, "--site", site, "--output-path", path.join(site, pagefindAssetDir)], { + stdio: "inherit", +}); +if (result.error) throw result.error; +if (result.status !== 0) process.exit(result.status ?? 1); diff --git a/scripts/docs-site/pagefind-cache.test.mjs b/scripts/docs-site/pagefind-cache.test.mjs new file mode 100644 index 0000000000..512699b9d9 --- /dev/null +++ b/scripts/docs-site/pagefind-cache.test.mjs @@ -0,0 +1,98 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import http from "node:http"; +import path from "node:path"; +import { test } from "node:test"; +import { chromium } from "playwright"; +import { fixture, repo, run, write } from "./test-helpers/redirect-fixture.mjs"; +import { pagefindAssetDir, pagefindVersion } from "./pagefind-assets.mjs"; + +test("Pagefind search uses the current build after immutable entry and worker assets were cached", async (t) => { + const f = fixture(t, [], { + "cron.md": "# Cron scheduler\n\nUse the cron scheduler to run recurring jobs.\n", + }); + write(f.root, "docs/docs.json", JSON.stringify({ + name: "Search fixture", + navigation: { languages: [{ language: "en", tabs: [{ tab: "Docs", groups: [{ group: "Guide", pages: ["index", "cron"] }] }] }] }, + })); + fs.symlinkSync(path.join(repo, "node_modules"), path.join(f.root, "node_modules"), process.platform === "win32" ? "junction" : "dir"); + const built = f.build(); + assert.equal(built.status, 0, built.stderr); + const site = path.join(f.root, "dist/docs-site"); + const pagefind = run(f.root, "pagefind-build.mjs"); + assert.equal(pagefind.status, 0, pagefind.stderr); + const normalized = run(f.root, "pagefind-normalize.mjs"); + assert.equal(normalized.status, 0, normalized.stderr); + assert.equal(JSON.parse(fs.readFileSync(path.join(site, pagefindAssetDir, "pagefind-entry.json"), "utf8")).version, pagefindVersion); + assert.ok(fs.readFileSync(path.join(site, "assets/docs-site.js"), "utf8").includes(`/${pagefindAssetDir}/pagefind.js`)); + for (const name of ["pagefind.js", "pagefind-worker.js"]) { + assert.ok(fs.readFileSync(path.join(site, pagefindAssetDir, name), "utf8").includes(`pagefind_version="${pagefindVersion}"`), + `${name} must match the generated index version`); + } + + // Model the reported failure boundary: the old entry imports and initializes, + // then its search rejects after reading the newer index, so import retry does + // not run. The worker is the real 1.5.2 output from a one-page index build. + const legacyEntry = 'export async function init() {}\nexport async function search() { throw new Error("Pagefind WASM Error (No pointer): 1.5.2 runtime, 1.6.0 index"); }\n'; + const legacyWorker = fs.readFileSync(new URL("./test-fixtures/pagefind-1.5.2/pagefind-worker.js", import.meta.url)); + let currentBuild = true; + const legacyCached = new Set(); + const currentRequests = []; + const server = http.createServer((req, res) => { + const url = new URL(req.url, "http://localhost"); + if (url.pathname === "/warm") return res.setHeader("Content-Type", "text/html").end("Previous visit"); + const legacyPath = url.pathname === "/pagefind/pagefind.js" || url.pathname === "/pagefind/pagefind-worker.js"; + const cacheKey = url.pathname + url.search; + if (legacyPath && (!currentBuild || legacyCached.has(cacheKey))) { + if (!currentBuild) legacyCached.add(cacheKey); + res.setHeader("Content-Type", "text/javascript"); + res.setHeader("Cache-Control", "public, max-age=31536000, immutable"); + return res.end(url.pathname.endsWith("-worker.js") ? legacyWorker : legacyEntry); + } + if (currentBuild) currentRequests.push(url.pathname); + const file = path.join(site, decodeURIComponent(url.pathname).replace(/^\/+/, "")); + if (!fs.existsSync(file)) return res.writeHead(404).end(); + const resolved = fs.statSync(file).isDirectory() ? path.join(file, "index.html") : file; + if (!fs.existsSync(resolved)) return res.writeHead(404).end(); + const contentType = ({ ".html": "text/html", ".js": "text/javascript", ".css": "text/css", ".json": "application/json", ".wasm": "application/wasm" })[path.extname(resolved)] || "application/octet-stream"; + res.setHeader("Content-Type", contentType); + res.setHeader("Cache-Control", "public, max-age=31536000, immutable"); + res.end(fs.readFileSync(resolved)); + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + t.after(() => new Promise(resolve => server.close(resolve))); + const browser = await chromium.launch({ headless: true }); + t.after(() => browser.close()); + const base = `http://127.0.0.1:${server.address().port}`; + + async function search(page) { + await page.goto(base + "/", { waitUntil: "domcontentloaded" }); + await page.locator("[data-search-open]").first().click(); + await page.locator("[data-search-input]").fill("cron scheduler"); + await page.locator(".search-result, .search-unavailable").first().waitFor({ timeout: 10000 }); + assert.match(await page.locator("[data-search-results]").innerText(), /Cron scheduler/i); + } + + const cold = await browser.newContext(); + t.after(() => cold.close()); + await search(await cold.newPage()); + currentRequests.length = 0; + currentBuild = false; + + const warm = await browser.newContext(); + t.after(() => warm.close()); + const warmPage = await warm.newPage(); + await warmPage.goto(base + "/warm"); + const workerStarted = warmPage.waitForEvent("worker"); + await warmPage.evaluate(async () => { + await import("/pagefind/pagefind.js"); + window.__legacyWorker = new Worker("/pagefind/pagefind-worker.js", { type: "module" }); + }); + await workerStarted; + await warmPage.evaluate(() => window.__legacyWorker.terminate()); + assert.deepEqual([...legacyCached].sort(), ["/pagefind/pagefind-worker.js", "/pagefind/pagefind.js"]); + currentBuild = true; + await search(warmPage); + assert.ok(currentRequests.some(url => /pagefind-worker\.js$/.test(url)), + "the upgraded search must request a worker outside the old immutable cache key"); +}); diff --git a/scripts/docs-site/pagefind-normalize.mjs b/scripts/docs-site/pagefind-normalize.mjs index 87b2e6a677..50fcbce264 100644 --- a/scripts/docs-site/pagefind-normalize.mjs +++ b/scripts/docs-site/pagefind-normalize.mjs @@ -1,13 +1,17 @@ #!/usr/bin/env node import fs from "node:fs"; import path from "node:path"; +import { pagefindAssetDir, pagefindVersion } from "./pagefind-assets.mjs"; const root = process.cwd(); -const entryPath = path.join(root, "dist", "docs-site", "pagefind", "pagefind-entry.json"); +const entryPath = path.join(root, "dist", "docs-site", pagefindAssetDir, "pagefind-entry.json"); if (!fs.existsSync(entryPath)) throw new Error("pagefind-entry.json does not exist; run pagefind first"); const entry = JSON.parse(fs.readFileSync(entryPath, "utf8")); +if (entry.version !== pagefindVersion) { + throw new Error(`Pagefind index version ${entry.version} does not match installed runtime ${pagefindVersion}`); +} if (entry.languages && typeof entry.languages === "object" && !Array.isArray(entry.languages)) { entry.languages = Object.fromEntries( Object.entries(entry.languages).sort(([left], [right]) => left.localeCompare(right)), diff --git a/scripts/docs-site/publication-stability.test.mjs b/scripts/docs-site/publication-stability.test.mjs index 8e86988a7b..29c50b4806 100644 --- a/scripts/docs-site/publication-stability.test.mjs +++ b/scripts/docs-site/publication-stability.test.mjs @@ -2,9 +2,8 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; -import { spawnSync } from "node:child_process"; import test from "node:test"; -import { fixture, repo, run, write } from "./test-helpers/redirect-fixture.mjs"; +import { fixture, run, write } from "./test-helpers/redirect-fixture.mjs"; import { siteJs } from "./site-js.mjs"; test("runtime identity hashes the executed source, including analytics and lazy dependencies", () => { @@ -31,11 +30,7 @@ test("commit-only builds preserve artifacts; two body edits change only their pa const result = run(f.root, script, env); assert.equal(result.status, 0, result.stderr); } - const pagefind = spawnSync(path.join(repo, "node_modules/.bin/pagefind"), ["--site", site], { - env: { PATH: process.env.PATH }, encoding: "utf8", - }); - assert.equal(pagefind.status, 0, pagefind.stderr); - for (const script of ["pagefind-normalize.mjs", "r2-prepare.mjs"]) { + for (const script of ["pagefind-build.mjs", "pagefind-normalize.mjs", "r2-prepare.mjs"]) { const result = run(f.root, script, env); assert.equal(result.status, 0, result.stderr); } diff --git a/scripts/docs-site/r2-prepare.mjs b/scripts/docs-site/r2-prepare.mjs index f0f07fba3b..afa8772dd2 100644 --- a/scripts/docs-site/r2-prepare.mjs +++ b/scripts/docs-site/r2-prepare.mjs @@ -111,7 +111,8 @@ function cacheControlFor(key) { if (key.endsWith(".html") || !path.extname(key)) { return "public, max-age=60, s-maxage=86400, stale-while-revalidate=604800"; } - if (key.endsWith(".md") || key.endsWith(".txt") || key.endsWith(".json") || key.endsWith(".jsonl")) { + if (key === "pagefind/pagefind.js" || key === "pagefind/pagefind-worker.js" + || key.endsWith(".md") || key.endsWith(".txt") || key.endsWith(".json") || key.endsWith(".jsonl")) { return "public, max-age=300, s-maxage=3600, stale-while-revalidate=86400"; } return "public, max-age=31536000, immutable"; diff --git a/scripts/docs-site/r2-upload.test.mjs b/scripts/docs-site/r2-upload.test.mjs index f8d6e7b698..49849d21c9 100644 --- a/scripts/docs-site/r2-upload.test.mjs +++ b/scripts/docs-site/r2-upload.test.mjs @@ -137,6 +137,23 @@ test("prepared artifact paths preserve uploaded bytes and reuse old manifest obj assert.deepEqual(puts(cold.stdout), prepared.entries.map((entry) => entry.key)); }); +test("R2 Pagefind metadata distinguishes legacy scripts from versioned build assets", (t) => { + const f = fixture(t); + const expected = new Map([ + ["pagefind/pagefind.js", "public, max-age=300, s-maxage=3600, stale-while-revalidate=86400"], + ["pagefind/pagefind-worker.js", "public, max-age=300, s-maxage=3600, stale-while-revalidate=86400"], + ["pagefind/v1.6.0/pagefind.js", "public, max-age=31536000, immutable"], + ["pagefind/v1.6.0/pagefind-worker.js", "public, max-age=31536000, immutable"], + ["pagefind/v1.6.0/pagefind-entry.json", "public, max-age=300, s-maxage=3600, stale-while-revalidate=86400"], + ["pagefind/v1.6.0/pagefind.en.pf_meta", "public, max-age=31536000, immutable"], + ]); + for (const key of expected.keys()) write(f.root, `dist/docs-site/${key}`, "fixture"); + const entries = new Map(f.prepare().entries.map((entry) => [entry.key, entry])); + for (const [key, cacheControl] of expected) { + assert.equal(entries.get(key)?.cacheControl, cacheControl, key); + } +}); + for (const mode of ["manifest", "head", "head-error"]) { test(`${mode} audit uploads metadata add/change/remove despite unchanged ETags using only mocked fetch`, (t) => { const f = uploadFixture(t); diff --git a/scripts/docs-site/site-js.mjs b/scripts/docs-site/site-js.mjs index 2133d26074..d4458a36d6 100644 --- a/scripts/docs-site/site-js.mjs +++ b/scripts/docs-site/site-js.mjs @@ -8,6 +8,7 @@ import { createDocsAnalytics } from "./analytics.mjs"; import { createDocsAnalyticsEvents } from "./analytics-events.mjs"; import { webVitalsAssetName } from "./web-vitals-runtime.mjs"; import { peleAssetName } from "./pele-runtime.mjs"; +import { pagefindAssetDir } from "./pagefind-assets.mjs"; import { initDocsAnalyticsConsent } from "./analytics-consent.mjs"; // The emitted runtime also supports the classic-script navigation harness. const heroRuntime = fs.readFileSync(new URL("./hero-art.mjs", import.meta.url), "utf8").replace(/^export /gm, ""); @@ -279,7 +280,7 @@ function currentTocHeadingId(headings){const top=parseFloat(getComputedStyle(doc function initTocScrollspy(){tocObserver?.disconnect();tocObserver=null;tocSizeObserver?.disconnect();tocSizeObserver=null;if(tocScrollHandler){removeEventListener("scroll",tocScrollHandler);tocScrollHandler=null;}const toc=document.querySelector(".toc");if(!toc)return;if("ResizeObserver" in window){tocSizeObserver=new ResizeObserver(syncTocSpace);tocSizeObserver.observe(toc);const nav=toc.querySelector("nav");if(nav)tocSizeObserver.observe(nav)}const links=[...toc.querySelectorAll('a[href^="#"]')];const headings=[...document.querySelectorAll(".doc h2[id],.doc h3[id]")].filter(heading=>links.some(link=>tocLinkId(link)===heading.id));links.forEach(link=>link.classList.remove("active"));if(!headings.length)return;const sync=()=>{if(Date.now(){cancelAnimationFrame(tocScrollRaf);tocScrollRaf=requestAnimationFrame(sync)};addEventListener("scroll",tocScrollHandler,{passive:true});requestAnimationFrame(sync);if(!("IntersectionObserver" in window)){sync();return}tocObserver=new IntersectionObserver(sync,{rootMargin:"-120px 0px -70% 0px",threshold:[0,1]});headings.forEach(heading=>tocObserver.observe(heading))} async function navigateTo(url,replace=false,animateHome=false){if(navigating)return false;navigating=true;if(!animateHome)homeGlimm.cancel();closeLanguage();closeMermaidOverlay();try{const res=await fetch(url.href,{credentials:"same-origin"});if(!res.ok||!res.headers.get("content-type")?.includes("text/html"))return false;const nextDoc=new DOMParser().parseFromString(await res.text(),"text/html");if(!nextDoc.querySelector(".main"))return false;const commit=()=>{disposeHomeHero();for(const name of ["docs-home","docs-home-layout"])document.body.classList.toggle(name,nextDoc.body.classList.contains(name));syncSidebar(nextDoc);swap(".header-left",nextDoc);swap(".main",nextDoc);initHomeHero();syncStickyHeaderOffset();syncTocDisclosure();initCodeGroups();initCodeFades();initAccordionMotion();initPageFeedback();initMermaid();document.title=nextDoc.title;history[replace?"replaceState":"pushState"]({docs:true},"",url.href);docsAnalytics.pageView();currentDocKey=location.pathname+location.search;setNavOpen(false);initTocScrollspy();scrollTarget(url.hash);syncCommunityInvite()};if(animateHome)await homeGlimm.run(commit);else commit();return true}catch{return false}finally{navigating=false}} function unavailableSearch(){return {unavailable:true,search:async()=>({results:[]})}} -function loadPagefind(){if(pagefindReady)return pagefindReady;const src=withBase("/pagefind/pagefind.js")+(pagefindLoadAttempt?"?retry="+pagefindLoadAttempt:"");pagefindLoadAttempt+=1;pagefindReady=import(src).then(m=>m.init?.().then?.(()=>m)??m).catch(()=>{pagefindReady=null;return unavailableSearch()});return pagefindReady} +function loadPagefind(){if(pagefindReady)return pagefindReady;const src=withBase(${JSON.stringify("/" + pagefindAssetDir + "/pagefind.js")})+(pagefindLoadAttempt?"?retry="+pagefindLoadAttempt:"");pagefindLoadAttempt+=1;pagefindReady=import(src).then(m=>m.init?.().then?.(()=>m)??m).catch(()=>{pagefindReady=null;return unavailableSearch()});return pagefindReady} let searchReturnFocus=null; function openSearch(){if(!modal)return;closeLanguage();if(document.body.classList.contains("nav-open"))setNavOpen(false);if(!modal.classList.contains("open")){searchReturnFocus=document.activeElement;docsTelemetry.popup("docs_search",true,"","search_dialog")}modal.classList.add("open");if(!searchQuery()&&results&&!results.innerHTML.trim())results.innerHTML=searchInitialHtml();setSearchChrome(searchQuery());setTimeout(()=>input?.focus(),0);loadPagefind()} function clearSearch(){docsTelemetry.cancelSearch();clearTimeout(searchTimer);seq+=1;searchActiveIndex=-1;setSearchLoading(false);if(input)input.value="";setSearchChrome("");if(results){results.classList.remove("is-loading");results.innerHTML=searchInitialHtml()}} diff --git a/scripts/docs-site/smoke.mjs b/scripts/docs-site/smoke.mjs index f93808c105..0ac1d2f6b7 100644 --- a/scripts/docs-site/smoke.mjs +++ b/scripts/docs-site/smoke.mjs @@ -8,6 +8,7 @@ import { DomUtils, parseDocument } from "htmlparser2"; import { ignoredDocDirs, localeFlags, localeLabels, navigationLocaleToDir } from "./config.mjs"; import { chromeStrings } from "./chrome-strings.mjs"; import { peleAssetName } from "./pele-runtime.mjs"; +import { pagefindAssetDir } from "./pagefind-assets.mjs"; import { homeStrings } from "./home-strings.mjs"; import { editSourceUrlForPage, frontmatterSourcePath, readSourceMetadata } from "./edit-source.mjs"; import { parseFrontmatter } from "../../.openclaw-sync/lib/docs-markdown.mjs"; @@ -51,7 +52,9 @@ if (!shellOnly) { ".well-known/llms.txt", "robots.txt", "sitemap.xml", - "pagefind/pagefind.js", + `${pagefindAssetDir}/pagefind.js`, + `${pagefindAssetDir}/pagefind-worker.js`, + `${pagefindAssetDir}/pagefind-entry.json`, "docs-search.json", ); } @@ -291,7 +294,7 @@ if (!/class="hljs-attr">channels<\/span>/.test(index) throw new Error("index: json5 config example was not syntax-highlighted"); } const modelsMarkdown = fs.readFileSync(path.join(site, "concepts/models.md"), "utf8"); -if (!/^---\nsummary: /m.test(modelsMarkdown) || !/title: "Models CLI"/m.test(modelsMarkdown)) { +if (!/^---\r?\nsummary: /m.test(modelsMarkdown) || !/title: "Models CLI"/m.test(modelsMarkdown)) { throw new Error("concepts/models.md: source markdown was not emitted"); } if (process.env.DOCS_SITE_BASE_PATH && (/src="\/assets\//.test(index) || /href="\/assets\//.test(index))) { diff --git a/scripts/docs-site/test-fixtures/pagefind-1.5.2/LICENSE.txt b/scripts/docs-site/test-fixtures/pagefind-1.5.2/LICENSE.txt new file mode 100644 index 0000000000..1a92e1c27d --- /dev/null +++ b/scripts/docs-site/test-fixtures/pagefind-1.5.2/LICENSE.txt @@ -0,0 +1,16 @@ +pagefind + +Copyright (c) Pagefind + +All rights reserved. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the ""Software""), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Includes other software related under the MIT license: +- vscode-ripgrep, Copyright Microsoft Corporation. For licensing see /LICENSE/LICENSE-vscode-ripgrep diff --git a/scripts/docs-site/test-fixtures/pagefind-1.5.2/pagefind-worker.js b/scripts/docs-site/test-fixtures/pagefind-1.5.2/pagefind-worker.js new file mode 100644 index 0000000000..e2b1dd2f33 --- /dev/null +++ b/scripts/docs-site/test-fixtures/pagefind-1.5.2/pagefind-worker.js @@ -0,0 +1,6 @@ +const pagefind_version="1.5.2";let wasm_bindgen=(function(exports){let script_src;if(typeof document!=='undefined'&&document.currentScript!==null){script_src=new URL("UNHANDLED",location.href).toString();}function add_synthetic_filter(ptr,filter){const ptr0=passStringToWasm0(filter,wasm.__wbindgen_malloc,wasm.__wbindgen_realloc);const len0=WASM_VECTOR_LEN;const ret=wasm.add_synthetic_filter(ptr,ptr0,len0);return ret>>>0;}exports.add_synthetic_filter=add_synthetic_filter;function enter_playground_mode(ptr){const ret=wasm.enter_playground_mode(ptr);return ret>>>0;}exports.enter_playground_mode=enter_playground_mode;function filters(ptr){let deferred1_0;let deferred1_1;try{const ret=wasm.filters(ptr);deferred1_0=ret[0];deferred1_1=ret[1];return getStringFromWasm0(ret[0],ret[1]);}finally{wasm.__wbindgen_free(deferred1_0,deferred1_1,1);}}exports.filters=filters;function init_pagefind(metadata_bytes){const ptr0=passArray8ToWasm0(metadata_bytes,wasm.__wbindgen_malloc);const len0=WASM_VECTOR_LEN;const ret=wasm.init_pagefind(ptr0,len0);return ret>>>0;}exports.init_pagefind=init_pagefind;function load_filter_chunk(ptr,chunk_bytes){const ptr0=passArray8ToWasm0(chunk_bytes,wasm.__wbindgen_malloc);const len0=WASM_VECTOR_LEN;const ret=wasm.load_filter_chunk(ptr,ptr0,len0);return ret>>>0;}exports.load_filter_chunk=load_filter_chunk;function load_index_chunk(ptr,chunk_bytes){const ptr0=passArray8ToWasm0(chunk_bytes,wasm.__wbindgen_malloc);const len0=WASM_VECTOR_LEN;const ret=wasm.load_index_chunk(ptr,ptr0,len0);return ret>>>0;}exports.load_index_chunk=load_index_chunk;function request_all_filter_indexes(ptr){let deferred1_0;let deferred1_1;try{const ret=wasm.request_all_filter_indexes(ptr);deferred1_0=ret[0];deferred1_1=ret[1];return getStringFromWasm0(ret[0],ret[1]);}finally{wasm.__wbindgen_free(deferred1_0,deferred1_1,1);}}exports.request_all_filter_indexes=request_all_filter_indexes;function request_filter_indexes(ptr,filters){let deferred2_0;let deferred2_1;try{const ptr0=passStringToWasm0(filters,wasm.__wbindgen_malloc,wasm.__wbindgen_realloc);const len0=WASM_VECTOR_LEN;const ret=wasm.request_filter_indexes(ptr,ptr0,len0);deferred2_0=ret[0];deferred2_1=ret[1];return getStringFromWasm0(ret[0],ret[1]);}finally{wasm.__wbindgen_free(deferred2_0,deferred2_1,1);}}exports.request_filter_indexes=request_filter_indexes;function request_indexes(ptr,query){let deferred2_0;let deferred2_1;try{const ptr0=passStringToWasm0(query,wasm.__wbindgen_malloc,wasm.__wbindgen_realloc);const len0=WASM_VECTOR_LEN;const ret=wasm.request_indexes(ptr,ptr0,len0);deferred2_0=ret[0];deferred2_1=ret[1];return getStringFromWasm0(ret[0],ret[1]);}finally{wasm.__wbindgen_free(deferred2_0,deferred2_1,1);}}exports.request_indexes=request_indexes;function search(ptr,query,original_query,filter,sort,exact,exact_diacritics){let deferred5_0;let deferred5_1;try{const ptr0=passStringToWasm0(query,wasm.__wbindgen_malloc,wasm.__wbindgen_realloc);const len0=WASM_VECTOR_LEN;const ptr1=passStringToWasm0(original_query,wasm.__wbindgen_malloc,wasm.__wbindgen_realloc);const len1=WASM_VECTOR_LEN;const ptr2=passStringToWasm0(filter,wasm.__wbindgen_malloc,wasm.__wbindgen_realloc);const len2=WASM_VECTOR_LEN;const ptr3=passStringToWasm0(sort,wasm.__wbindgen_malloc,wasm.__wbindgen_realloc);const len3=WASM_VECTOR_LEN;const ret=wasm.search(ptr,ptr0,len0,ptr1,len1,ptr2,len2,ptr3,len3,exact,exact_diacritics);deferred5_0=ret[0];deferred5_1=ret[1];return getStringFromWasm0(ret[0],ret[1]);}finally{wasm.__wbindgen_free(deferred5_0,deferred5_1,1);}}exports.search=search;function set_ranking_weights(ptr,weights){const ptr0=passStringToWasm0(weights,wasm.__wbindgen_malloc,wasm.__wbindgen_realloc);const len0=WASM_VECTOR_LEN;const ret=wasm.set_ranking_weights(ptr,ptr0,len0);return ret>>>0;}exports.set_ranking_weights=set_ranking_weights;function __wbg_get_imports(){const import0={__proto__:null,__wbindgen_init_externref_table:function(){const table=wasm.__wbindgen_externrefs;const offset=table.grow(4);table.set(0,undefined);table.set(offset+0,undefined);table.set(offset+1,null);table.set(offset+2,true);table.set(offset+3,false);},};return{__proto__:null,"./pagefind_web_bg.js":import0,};}function getStringFromWasm0(ptr,len){ptr=ptr>>>0;return decodeText(ptr,len);}let cachedUint8ArrayMemory0=null;function getUint8ArrayMemory0(){if(cachedUint8ArrayMemory0===null||cachedUint8ArrayMemory0.byteLength===0){cachedUint8ArrayMemory0=new Uint8Array(wasm.memory.buffer);}return cachedUint8ArrayMemory0;}function passArray8ToWasm0(arg,malloc){const ptr=malloc(arg.length*1,1)>>>0;getUint8ArrayMemory0().set(arg,ptr/1);WASM_VECTOR_LEN=arg.length;return ptr;}function passStringToWasm0(arg,malloc,realloc){if(realloc===undefined){const buf=cachedTextEncoder.encode(arg);const ptr=malloc(buf.length,1)>>>0;getUint8ArrayMemory0().subarray(ptr,ptr+buf.length).set(buf);WASM_VECTOR_LEN=buf.length;return ptr;}let len=arg.length;let ptr=malloc(len,1)>>>0;const mem=getUint8ArrayMemory0();let offset=0;for(;offset0x7F)break;mem[ptr+offset]=code;}if(offset!==len){if(offset!==0){arg=arg.slice(offset);}ptr=realloc(ptr,len,len=offset+arg.length*3,1)>>>0;const view=getUint8ArrayMemory0().subarray(ptr+offset,ptr+len);const ret=cachedTextEncoder.encodeInto(arg,view);offset+=ret.written;ptr=realloc(ptr,len,offset,1)>>>0;}WASM_VECTOR_LEN=offset;return ptr;}let cachedTextDecoder=new TextDecoder('utf-8',{ignoreBOM:true,fatal:true});cachedTextDecoder.decode();function decodeText(ptr,len){return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr,ptr+len));}const cachedTextEncoder=new TextEncoder();if(!('encodeInto'in cachedTextEncoder)){cachedTextEncoder.encodeInto=function(arg,view){const buf=cachedTextEncoder.encode(arg);view.set(buf);return{read:arg.length,written:buf.length};};}let WASM_VECTOR_LEN=0;let wasmModule,wasm;function __wbg_finalize_init(instance,module){wasm=instance.exports;wasmModule=module;cachedUint8ArrayMemory0=null;wasm.__wbindgen_start();return wasm;}async function __wbg_load(module,imports){if(typeof Response==='function'&&module instanceof Response){if(typeof WebAssembly.instantiateStreaming==='function'){try{return await WebAssembly.instantiateStreaming(module,imports);}catch(e){const validResponse=module.ok&&expectedResponseType(module.type);if(validResponse&&module.headers.get('Content-Type')!=='application/wasm'){console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n",e);}else{throw e;}}}const bytes=await module.arrayBuffer();return await WebAssembly.instantiate(bytes,imports);}else{const instance=await WebAssembly.instantiate(module,imports);if(instance instanceof WebAssembly.Instance){return{instance,module};}else{return instance;}}function expectedResponseType(type){switch(type){case'basic':case'cors':case'default':return true;}return false;}}function initSync(module){if(wasm!==undefined)return wasm;if(module!==undefined){if(Object.getPrototypeOf(module)===Object.prototype){({module}=module)}else{console.warn('using deprecated parameters for `initSync()`; pass a single object instead')}}const imports=__wbg_get_imports();if(!(module instanceof WebAssembly.Module)){module=new WebAssembly.Module(module);}const instance=new WebAssembly.Instance(module,imports);return __wbg_finalize_init(instance,module);}async function __wbg_init(module_or_path){if(wasm!==undefined)return wasm;if(module_or_path!==undefined){if(Object.getPrototypeOf(module_or_path)===Object.prototype){({module_or_path}=module_or_path)}else{console.warn('using deprecated parameters for the initialization function; pass a single object instead')}}if(module_or_path===undefined&&script_src!==undefined){module_or_path=script_src.replace(/\.js$/,"_bg.wasm");}const imports=__wbg_get_imports();if(typeof module_or_path==='string'||(typeof Request==='function'&&module_or_path instanceof Request)||(typeof URL==='function'&&module_or_path instanceof URL)){module_or_path=fetch(module_or_path);}const{instance,module}=await __wbg_load(await module_or_path,imports);return __wbg_finalize_init(instance,module);}return Object.assign(__wbg_init,{initSync},exports);})({__proto__:null});"use strict";(()=>{var __defProp=Object.defineProperty;var __defNormalProp=(obj,key,value)=>key in obj?__defProp(obj,key,{enumerable:true,configurable:true,writable:true,value}):obj[key]=value;var __publicField=(obj,key,value)=>__defNormalProp(obj,typeof key!=="symbol"?key+"":key,value);var u8=Uint8Array;var u16=Uint16Array;var u32=Uint32Array;var fleb=new u8([0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0,0]);var fdeb=new u8([0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,0,0]);var clim=new u8([16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15]);var freb=function(eb,start){var b=new u16(31);for(var i2=0;i2<31;++i2){b[i2]=start+=1<>>1|(i&21845)<<1;x=(x&52428)>>>2|(x&13107)<<2;x=(x&61680)>>>4|(x&3855)<<4;rev[i]=((x&65280)>>>8|(x&255)<<8)>>>1;}var x;var i;var hMap=function(cd,mb,r){var s=cd.length;var i2=0;var l=new u16(mb);for(;i2>>rvb]=sv;}}}}else{co=new u16(s);for(i2=0;i2>>15-cd[i2];}}}return co;};var flt=new u8(288);for(i=0;i<144;++i)flt[i]=8;var i;for(i=144;i<256;++i)flt[i]=9;var i;for(i=256;i<280;++i)flt[i]=7;var i;for(i=280;i<288;++i)flt[i]=8;var i;var fdt=new u8(32);for(i=0;i<32;++i)fdt[i]=5;var i;var flrm=hMap(flt,9,1);var fdrm=hMap(fdt,5,1);var max=function(a){var m=a[0];for(var i2=1;i2m)m=a[i2];}return m;};var bits=function(d,p,m){var o=p/8|0;return(d[o]|d[o+1]<<8)>>(p&7)&m;};var bits16=function(d,p){var o=p/8|0;return(d[o]|d[o+1]<<8|d[o+2]<<16)>>(p&7);};var shft=function(p){return(p+7)/8|0;};var slc=function(v,s,e){if(s==null||s<0)s=0;if(e==null||e>v.length)e=v.length;var n=new(v.BYTES_PER_ELEMENT==2?u16:v.BYTES_PER_ELEMENT==4?u32:u8)(e-s);n.set(v.subarray(s,e));return n;};var ec=["unexpected EOF","invalid block type","invalid length/literal","invalid distance","stream finished","no stream handler",,"no callback","invalid UTF-8 data","extra field too long","date not in range 1980-2099","filename too long","stream finishing","invalid zip data"];var err=function(ind,msg,nt){var e=new Error(msg||ec[ind]);e.code=ind;if(Error.captureStackTrace)Error.captureStackTrace(e,err);if(!nt)throw e;return e;};var inflt=function(dat,buf,st){var sl=dat.length;if(!sl||st&&st.f&&!st.l)return buf||new u8(0);var noBuf=!buf||st;var noSt=!st||st.i;if(!st)st={};if(!buf)buf=new u8(sl*3);var cbuf=function(l2){var bl=buf.length;if(l2>bl){var nbuf=new u8(Math.max(bl*2,l2));nbuf.set(buf);buf=nbuf;}};var final=st.f||0,pos=st.p||0,bt=st.b||0,lm=st.l,dm=st.d,lbt=st.m,dbt=st.n;var tbts=sl*8;do{if(!lm){final=bits(dat,pos,1);var type=bits(dat,pos+1,3);pos+=3;if(!type){var s=shft(pos)+4,l=dat[s-4]|dat[s-3]<<8,t=s+l;if(t>sl){if(noSt)err(0);break;}if(noBuf)cbuf(bt+l);buf.set(dat.subarray(s,t),bt);st.b=bt+=l,st.p=pos=t*8,st.f=final;continue;}else if(type==1)lm=flrm,dm=fdrm,lbt=9,dbt=5;else if(type==2){var hLit=bits(dat,pos,31)+257,hcLen=bits(dat,pos+10,15)+4;var tl=hLit+bits(dat,pos+5,31)+1;pos+=14;var ldt=new u8(tl);var clt=new u8(19);for(var i2=0;i2>>4;if(s<16){ldt[i2++]=s;}else{var c=0,n=0;if(s==16)n=3+bits(dat,pos,3),pos+=2,c=ldt[i2-1];else if(s==17)n=3+bits(dat,pos,7),pos+=3;else if(s==18)n=11+bits(dat,pos,127),pos+=7;while(n--)ldt[i2++]=c;}}var lt=ldt.subarray(0,hLit),dt=ldt.subarray(hLit);lbt=max(lt);dbt=max(dt);lm=hMap(lt,lbt,1);dm=hMap(dt,dbt,1);}else err(1);if(pos>tbts){if(noSt)err(0);break;}}if(noBuf)cbuf(bt+131072);var lms=(1<>>4;pos+=c&15;if(pos>tbts){if(noSt)err(0);break;}if(!c)err(2);if(sym<256)buf[bt++]=sym;else if(sym==256){lpos=pos,lm=null;break;}else{var add=sym-254;if(sym>264){var i2=sym-257,b=fleb[i2];add=bits(dat,pos,(1<>>4;if(!d)err(3);pos+=d&15;var dt=fd[dsym];if(dsym>3){var b=fdeb[dsym];dt+=bits16(dat,pos)&(1<tbts){if(noSt)err(0);break;}if(noBuf)cbuf(bt+131072);var end=bt+add;for(;bt>3&1)+(flg>>4&1);zs>0;zs-=!d[st++]);return st+(flg&2);};var gzl=function(d){var l=d.length;return(d[l-4]|d[l-3]<<8|d[l-2]<<16|d[l-1]<<24)>>>0;};function gunzipSync(data,out){return inflt(data.subarray(gzs(data),-8),out||new u8(gzl(data)));}var td=typeof TextDecoder!="undefined"&&new TextDecoder();var tds=0;try{td.decode(et,{stream:true});tds=1;}catch(e){}var gz_default=gunzipSync;var calculate_excerpt_region=(word_positions,excerpt_length)=>{if(word_positions.length===0){return 0;}let words=[];for(const word of word_positions){words[word.location]=words[word.location]||0;words[word.location]+=word.balanced_score;}if(words.length<=excerpt_length){return 0;}let densest=words.slice(0,excerpt_length).reduce((partialSum,a)=>partialSum+a,0);let working_sum=densest;let densest_at=[0];for(let i2=0;i2densest){densest=working_sum;densest_at=[i2];}else if(working_sum===densest&&densest_at[densest_at.length-1]===i2-1){densest_at.push(i2);}}let midpoint=densest_at[Math.floor(densest_at.length/2)];return midpoint;};var build_excerpt=(content,start,length,locations,not_before,not_from)=>{let is_zws_delimited=content.includes("\u200B");let fragment_words=[];if(is_zws_delimited){fragment_words=content.split("\u200B");}else{fragment_words=content.split(/[\r\n\s]+/g);}let endcap=not_from??fragment_words.length;let startcap=not_before??0;if(endcap-startcapendcap){start=endcap-length;}if(start`)){continue;}fragment_words[word]=`${fragment_words[word]}`;}const excerpt=fragment_words.slice(start,start+length).join(joiner).trim();return{excerpt,plain_excerpt};};var calculate_sub_results=(fragment,desired_excerpt_length)=>{const effective_url=fragment.meta?.url||fragment.url;const anchors=fragment.anchors.filter((a)=>/h\d/i.test(a.element)&&a.text?.length&&/\S/.test(a.text)).sort((a,b)=>a.location-b.location);const results=[];let current_anchor_position=0;let current_anchor={title:fragment.meta["title"],url:effective_url,weighted_locations:[],locations:[],excerpt:"",plain_excerpt:""};const add_result=(end_range)=>{if(current_anchor.locations.length){const relative_weighted_locations=current_anchor.weighted_locations.map((l)=>{return{weight:l.weight,balanced_score:l.balanced_score,location:l.location-current_anchor_position};});const excerpt_start=calculate_excerpt_region(relative_weighted_locations,desired_excerpt_length)+current_anchor_position;const excerpt_length=end_range?Math.min(end_range-excerpt_start,desired_excerpt_length):desired_excerpt_length;const excerpts=build_excerpt(fragment.raw_content??"",excerpt_start,excerpt_length,current_anchor.locations,current_anchor_position,end_range);current_anchor.excerpt=excerpts.excerpt;current_anchor.plain_excerpt=excerpts.plain_excerpt;results.push(current_anchor);}};for(let word of fragment.weighted_locations){if(!anchors.length||word.location=anchors[0].location){next_anchor=anchors.shift();}let anchored_url=effective_url;try{const url_is_fq=/^((https?:)?\/\/)/.test(anchored_url);if(url_is_fq){let fq_url=new URL(anchored_url);fq_url.hash=next_anchor.id;anchored_url=fq_url.toString();}else{if(!/^\//.test(anchored_url)){anchored_url=`/${anchored_url}`;}let fq_url=new URL(`https://example.com${anchored_url}`);fq_url.hash=next_anchor.id;anchored_url=fq_url.toString().replace(/^https:\/\/example.com/,"");}}catch(e){console.error(`Pagefind: Couldn't process ${anchored_url} for a search result`);}current_anchor_position=next_anchor.location;current_anchor={title:next_anchor.text,url:anchored_url,anchor:next_anchor,weighted_locations:[word],locations:[word.location],excerpt:"",plain_excerpt:""};}}add_result(anchors[0]?.location);return results;};var import_meta={};var asyncSleep=async(ms=100)=>{return new Promise((r)=>setTimeout(r,ms));};var normalizeDiacritics=(str)=>{return str.normalize("NFD").replace(/\p{M}/gu,"");};var isBrowser=()=>typeof window!=="undefined"&&typeof document!=="undefined";var needsWordSegmentation=(lang)=>{if(!lang)return false;const primaryLang=lang.split("-")[0].toLowerCase();return["zh","ja","th"].includes(primaryLang);};var PagefindInstance=class{constructor(opts={}){__publicField(this,"backend");__publicField(this,"decoder");__publicField(this,"wasm");__publicField(this,"basePath");__publicField(this,"baseUrl");__publicField(this,"primary");__publicField(this,"indexWeight");__publicField(this,"excerptLength");__publicField(this,"mergeFilter");__publicField(this,"ranking");__publicField(this,"highlightParam");__publicField(this,"exactDiacritics");__publicField(this,"metaCacheTag");__publicField(this,"loaded_chunks");__publicField(this,"loaded_filters");__publicField(this,"loaded_fragments");__publicField(this,"fetchQueue",[]);__publicField(this,"activeFetches",0);__publicField(this,"maxConcurrentFetches",100);__publicField(this,"raw_ptr");__publicField(this,"initError");__publicField(this,"searchMeta");__publicField(this,"languages");__publicField(this,"loadedLanguage");__publicField(this,"includeCharacters");__publicField(this,"version");__publicField(this,"loadedVersion");this.version=pagefind_version;this.backend=wasm_bindgen;this.decoder=new TextDecoder("utf-8");this.wasm=null;let basePath=opts.basePath||"/pagefind/";let primary=opts.primary||false;if(primary&&!opts.basePath&&isBrowser()){basePath=this.initPrimaryBasePath(basePath);}if(/[^\/]$/.test(basePath)){basePath=`${basePath}/`;}if(isBrowser()&&window?.location?.origin&&basePath.startsWith(window.location.origin)){basePath=basePath.replace(window.location.origin,"");}this.basePath=basePath;this.baseUrl=opts.baseUrl||this.getDefaultBaseUrl(basePath);if(!/^(\/|https?:\/\/)/.test(this.baseUrl)){this.baseUrl=`/${this.baseUrl}`;}this.primary=primary;this.indexWeight=opts.indexWeight??1;this.excerptLength=opts.excerptLength??30;this.mergeFilter=opts.mergeFilter??{};this.ranking=opts.ranking;this.highlightParam=opts.highlightParam??null;this.exactDiacritics=opts.exactDiacritics??false;this.metaCacheTag=opts.metaCacheTag??null;this.loaded_chunks={};this.loaded_filters={};this.loaded_fragments={};this.raw_ptr=null;this.initError=null;this.searchMeta=null;this.languages=null;}throttledFetch(input){return new Promise((resolve,reject)=>{this.fetchQueue.push({resolve,reject,input});this.dequeueNextFetch();});}dequeueNextFetch(){while(this.fetchQueue.length>0&&this.activeFetchesb.page_count-a.page_count);if(topLang[0])return topLang[0];}throw new Error("Pagefind Error: No language indexes found.");}async loadMeta(index){try{let compressed_resp=await this.throttledFetch(`${this.basePath}pagefind.${index}.pf_meta`);let compressed_meta=await compressed_resp.arrayBuffer();this.searchMeta=this.decompress(new Uint8Array(compressed_meta),"Pagefind metadata");}catch(e){console.error(`Failed to load the meta index: +${e?.toString()}`);}}async loadWasm(language){try{const wasm_url=`${this.basePath}wasm.${language}.pagefind`;let compressed_resp=await this.throttledFetch(wasm_url);let compressed_wasm=await compressed_resp.arrayBuffer();const final_wasm=this.decompress(new Uint8Array(compressed_wasm),"Pagefind WebAssembly");if(!final_wasm){throw new Error("No WASM after decompression");}this.wasm=await this.backend({module_or_path:final_wasm});}catch(e){console.error(`Failed to load the Pagefind WASM: +${e?.toString()}`);throw new Error(`Failed to load the Pagefind WASM: +${e?.toString()}`);}}async _loadGenericChunk(url,method){try{let compressed_resp=await this.throttledFetch(url);let compressed_chunk=await compressed_resp.arrayBuffer();let chunk=this.decompress(new Uint8Array(compressed_chunk),url);let ptr=await this.getPtr();this.raw_ptr=this.backend[method](ptr,chunk);}catch(e){console.error(`Failed to load the index chunk ${url}: +${e?.toString()}`);}}async loadChunk(hash){if(!this.loaded_chunks[hash]){const url=`${this.basePath}index/${hash}.pf_index`;this.loaded_chunks[hash]=this._loadGenericChunk(url,"load_index_chunk");}return await this.loaded_chunks[hash];}async loadFilterChunk(hash){if(!this.loaded_filters[hash]){const url=`${this.basePath}filter/${hash}.pf_filter`;this.loaded_filters[hash]=this._loadGenericChunk(url,"load_filter_chunk");}return await this.loaded_filters[hash];}async _loadFragment(hash){let compressed_resp=await this.throttledFetch(`${this.basePath}fragment/${hash}.pf_fragment`);let compressed_fragment=await compressed_resp.arrayBuffer();let fragment=this.decompress(new Uint8Array(compressed_fragment),`Fragment ${hash}`);return JSON.parse(new TextDecoder().decode(fragment));}async loadFragment(hash,weighted_locations=[],search_term){if(!this.loaded_fragments[hash]){this.loaded_fragments[hash]=this._loadFragment(hash);}let fragment=await this.loaded_fragments[hash];fragment.weighted_locations=weighted_locations;fragment.locations=weighted_locations.map((l)=>l.location);if(!fragment.raw_content){fragment.raw_content=fragment.content.replace(//g,">");fragment.content=fragment.content.replace(/\u200B/g,"");}if(!fragment.raw_url){fragment.raw_url=fragment.url;}fragment.url=this.processedUrl(fragment.raw_url,search_term);const excerpt_start=calculate_excerpt_region(weighted_locations,this.excerptLength);const excerpts=build_excerpt(fragment.raw_content,excerpt_start,this.excerptLength,fragment.locations);fragment.excerpt=excerpts.excerpt;fragment.plain_excerpt=excerpts.plain_excerpt;fragment.sub_results=calculate_sub_results(fragment,this.excerptLength);return fragment;}fullUrl(raw){if(/^(https?:)?\/\//.test(raw)){return raw;}return`${this.baseUrl}/${raw}`.replace(/\/+/g,"/").replace(/^(https?:\/)/,"$1/");}processedUrl(url,search_term){const normalized=this.fullUrl(url);if(this.highlightParam===null){return normalized;}let individual_terms=search_term.split(/\s+/);try{let processed=new URL(normalized);for(const term of individual_terms){processed.searchParams.append(this.highlightParam,term);}return processed.toString();}catch(e){try{let processed=new URL(`https://example.com${normalized}`);for(const term of individual_terms){processed.searchParams.append(this.highlightParam,term);}return processed.toString().replace(/^https:\/\/example\.com/,"");}catch(e2){return normalized;}}}async getPtr(){while(this.raw_ptr===null){if(this.initError){throw this.initError;}await asyncSleep(50);}if(!this.raw_ptr){console.error("Pagefind: WASM Error (No pointer)");throw new Error("Pagefind: WASM Error (No pointer)");}return this.raw_ptr;}stringifyFilters(obj={}){return JSON.stringify(obj);}stringifySorts(obj={}){let sorts=Object.entries(obj);for(let[sort,direction]of sorts){if(sorts.length>1){console.warn(`Pagefind was provided multiple sort options in this search, but can only operate on one. Using the ${sort} sort.`);}if(direction!=="asc"&&direction!=="desc"){console.warn(`Pagefind was provided a sort with unknown direction ${direction}. Supported: [asc, desc]`);}return`${sort}:${direction}`;}return``;}async filters(){let ptr=await this.getPtr();let filters=this.backend.request_all_filter_indexes(ptr);let filter_array=JSON.parse(filters);if(Array.isArray(filter_array)){let filter_chunks=filter_array.filter((v)=>v).map((chunk)=>this.loadFilterChunk(chunk));await Promise.all([...filter_chunks]);}ptr=await this.getPtr();let results=this.backend.filters(ptr);return JSON.parse(results);}async preload(term,options={}){await this.search(term,{...options,preload:true});}async search(term,options={}){options={verbose:false,filters:{},sort:{},...options};const log=(str)=>{if(options.verbose)console.log(str);};log(`Starting search on ${this.basePath}`);let start=Date.now();let ptr=await this.getPtr();let filter_only=term===null;term=term??"";let exact_search=/^\s*".+"\s*$/.test(term);if(exact_search){log(`Running an exact search`);}let trueLanguage=null;try{trueLanguage=Intl.getCanonicalLocales(this.loadedLanguage)[0];}catch(err2){}const term_chunks=[];if(trueLanguage&&typeof Intl.Segmenter!=="undefined"){const graphemeSegmenter=new Intl.Segmenter(trueLanguage,{granularity:"grapheme"});if(needsWordSegmentation(trueLanguage)){const wordSegmenter=new Intl.Segmenter(trueLanguage,{granularity:"word"});for(const{segment:word}of wordSegmenter.segment(term)){const wordChunks=[];for(const{segment:grapheme}of graphemeSegmenter.segment(word)){if(this.includeCharacters?.includes(grapheme)){wordChunks.push(grapheme);}else if(!/^\p{Pd}|\p{Pe}|\p{Pf}|\p{Pi}|\p{Po}|\p{Ps}$/u.test(grapheme)){wordChunks.push(grapheme.toLocaleLowerCase());}}if(wordChunks.length>0){term_chunks.push(wordChunks.join(""));}}term=term_chunks.join(" ").replace(/\s{2,}/g," ").trim();}else{for(const{segment:grapheme}of graphemeSegmenter.segment(term)){if(this.includeCharacters?.includes(grapheme)){term_chunks.push(grapheme);}else if(!/^\p{Pd}|\p{Pe}|\p{Pf}|\p{Pi}|\p{Po}|\p{Ps}$/u.test(grapheme)){term_chunks.push(grapheme.toLocaleLowerCase());}}term=term_chunks.join("").replace(/\s{2,}/g," ").trim();}}else{for(const char of term){if(this.includeCharacters?.includes(char)){term_chunks.push(char);}else if(!/^\p{Pd}|\p{Pe}|\p{Pf}|\p{Pi}|\p{Po}|\p{Ps}$/u.test(char)){term_chunks.push(char.toLocaleLowerCase());}}term=term_chunks.join("").replace(/\s{2,}/g," ").trim();}const originalTerm=term;term=normalizeDiacritics(term);log(`Normalized search term to ${term}`);if(!term?.length&&!filter_only){return{results:[],unfilteredResultCount:0,filters:{},totalFilters:{},timings:{preload:Date.now()-start,search:Date.now()-start,total:Date.now()-start}};}let sort_list=this.stringifySorts(options.sort);log(`Stringified sort to ${sort_list}`);const filter_list=this.stringifyFilters(options.filters);log(`Stringified filters to ${filter_list}`);let index_resp=this.backend.request_indexes(ptr,term);let index_array=JSON.parse(index_resp);let filter_resp=this.backend.request_filter_indexes(ptr,filter_list);let filter_array=JSON.parse(filter_resp);let chunks=index_array.filter((v)=>v).map((chunk)=>this.loadChunk(chunk));let filter_chunks=filter_array.filter((v)=>v).map((chunk)=>this.loadFilterChunk(chunk));await Promise.all([...chunks,...filter_chunks]);log(`Loaded necessary chunks to run search`);if(options.preload){log(`Preload \u2014 bailing out of search operation now.`);return null;}ptr=await this.getPtr();let searchStart=Date.now();let result=this.backend.search(ptr,term,originalTerm,filter_list,sort_list,exact_search,this.exactDiacritics);log(`Got the raw search result: ${result}`);let{filtered_counts,total_counts,results,unfiltered_total,search_keywords,query_term_idfs}=JSON.parse(result);let resultsInterface=results.map((result2)=>{let weighted_locations=result2.l.map((l)=>{let loc={weight:l.w/24,balanced_score:l.s,location:l.l};if(l.v){loc.verbose={word_string:l.v.ws,length_bonus:l.v.lb};}return loc;});let locations=weighted_locations.map((l)=>l.location);let res={id:result2.p,score:result2.s*this.indexWeight,words:locations,data:async()=>await this.loadFragment(result2.p,weighted_locations,term)};if(result2.params){res.params={document_length:result2.params.dl,average_page_length:result2.params.apl,total_pages:result2.params.tp};}if(result2.scores){res.scores=result2.scores.map((r)=>{return{search_term:r.w,idf:r.idf,saturating_tf:r.b_tf,raw_tf:r.r_tf,pagefind_tf:r.p_tf,score:r.s,params:{weighted_term_frequency:r.params.w_tf,pages_containing_term:r.params.pct,length_bonus:r.params.lb}};});}if(result2.mf&&result2.mf.length>0){res.matchedMetaFields=result2.mf;}if(result2.vms&&result2.vms.length>0){res.verbose_meta_scores=result2.vms.map((s)=>({field_name:s.fn,field_weight:s.fw,matched_terms:s.mt,matched_idf:s.mi,query_total_idf:s.ti,coverage:s.cv,coverage_boost:s.cb}));}return res;});const searchTime=Date.now()-searchStart;const realTime=Date.now()-start;log(`Found ${results.length} result${results.length == 1 ? "" : "s"} for "${term}" in ${Date.now() - searchStart}ms (${Date.now() - start}ms realtime)`);let response={results:resultsInterface,unfilteredResultCount:unfiltered_total,filters:filtered_counts,totalFilters:total_counts,timings:{preload:realTime-searchTime,search:searchTime,total:realTime}};if(search_keywords){response.search_keywords=search_keywords;}if(query_term_idfs){response.query_term_idfs=query_term_idfs.map((q)=>({term:q.t,idf:q.i}));}return response;}};var Pagefind=class{constructor(options={}){__publicField(this,"primaryLanguage");__publicField(this,"searchID");__publicField(this,"primary");__publicField(this,"instances");this.primaryLanguage="unknown";this.searchID=0;this.primary=new PagefindInstance({...options,primary:true});this.instances=[this.primary];this.init(options?.language);}async options(options){await this.primary.options(options);}async enterPlaygroundMode(){await this.primary.enterPlaygroundMode();}async init(overrideLanguage){if(isBrowser()&&document?.querySelector){const langCode=document.querySelector("html")?.getAttribute("lang")||"unknown";this.primaryLanguage=langCode.toLocaleLowerCase();}if(overrideLanguage){this.primaryLanguage=overrideLanguage;}await this.primary.init(overrideLanguage?overrideLanguage:this.primaryLanguage,{load_wasm:true});}async mergeIndex(indexPath,options={}){if(this.primary.basePath.startsWith(indexPath)){console.warn(`Skipping mergeIndex ${indexPath} that appears to be the same as the primary index (${this.primary.basePath})`);return;}let newInstance=new PagefindInstance({primary:false,basePath:indexPath,...options});this.instances.push(newInstance);while(this.primary.wasm===null){await asyncSleep(50);}await newInstance.init(options.language||this.primaryLanguage,{load_wasm:false});const{language,...remainingOptions}=options;await newInstance.options(remainingOptions);}mergeFilters(filters){const merged={};for(const searchFilter of filters){for(const[filterKey,values]of Object.entries(searchFilter)){if(!merged[filterKey]){merged[filterKey]=values;continue;}else{const filter=merged[filterKey];for(const[valueKey,count]of Object.entries(values)){filter[valueKey]=(filter[valueKey]||0)+count;}}}}return merged;}async filters(){let filters=await Promise.all(this.instances.map((i2)=>i2.filters()));return this.mergeFilters(filters);}async preload(term,options={}){await Promise.all(this.instances.map((i2)=>i2.preload(term,options)));}async debouncedSearch(term,options,debounceTimeoutMs){const thisSearchID=++this.searchID;this.preload(term,options);await asyncSleep(debounceTimeoutMs);if(thisSearchID!==this.searchID){return null;}const searchResult=await this.search(term,options);if(thisSearchID!==this.searchID){return null;}return searchResult;}async search(term,options={}){let search=await Promise.all(this.instances.map((i2)=>i2.search(term,options)));const filters=this.mergeFilters(search.map((s)=>s.filters));const totalFilters=this.mergeFilters(search.map((s)=>s.totalFilters));const results=search.map((s)=>s.results).flat().sort((a,b)=>b.score-a.score);const timings=search.map((s)=>s.timings);const unfilteredResultCount=search.reduce((sum,s)=>sum+s.unfilteredResultCount,0);let response={results,unfilteredResultCount,filters,totalFilters,timings};if(search[0].search_keywords){response.search_keywords=search[0].search_keywords;}if(search[0].query_term_idfs){response.query_term_idfs=search[0].query_term_idfs;}return response;}};var dataCallbacks=new Map();var instanceDataIds=new Map();var instances=new Map();var DEFAULT_INSTANCE="default";var getInstance=(instanceId)=>{const instance=instances.get(instanceId);if(!instance){throw new Error(`Pagefind instance "${instanceId}" not initialized`);}return instance;};var registerDataCallback=(instanceId,dataId,dataFn)=>{dataCallbacks.set(dataId,{getData:dataFn});if(!instanceDataIds.has(instanceId)){instanceDataIds.set(instanceId,new Set());}instanceDataIds.get(instanceId).add(dataId);};var handleMessage=async(message)=>{const{id,method,args}=message;const instanceId=message.instanceId??DEFAULT_INSTANCE;try{switch(method){case"init":{const[options]=args;instances.set(instanceId,new Pagefind(options));return{id,result:true};}case"options":{const pagefindInstance=getInstance(instanceId);const[options]=args;await pagefindInstance.options(options);return{id,result:true};}case"enterPlaygroundMode":{const pagefindInstance=getInstance(instanceId);await pagefindInstance.enterPlaygroundMode();return{id,result:true};}case"mergeIndex":{const pagefindInstance=getInstance(instanceId);const[indexPath,options]=args;await pagefindInstance.mergeIndex(indexPath,options);return{id,result:true};}case"search":{const pagefindInstance=getInstance(instanceId);const[term,options]=args;const results=await pagefindInstance.search(term,options);if(results&&results.results){for(let i2=0;i2{const message=event.data;const response=await handleMessage(message);self.postMessage(response);});})(); \ No newline at end of file diff --git a/workers/docs-router-dotted.test.mjs b/workers/docs-router-dotted.test.mjs index 0c5e59ea23..021e16c54b 100644 --- a/workers/docs-router-dotted.test.mjs +++ b/workers/docs-router-dotted.test.mjs @@ -189,6 +189,22 @@ const assets = [ ["assets/docs-site.css", "text/css; charset=utf-8", "body{}", "public, max-age=60, stale-while-revalidate=300", mutableEdge], ["assets/docs-site.js", "text/javascript; charset=utf-8", "export {};", "public, max-age=60, stale-while-revalidate=300", mutableEdge], ]; + +test("Pagefind legacy scripts are mutable while versioned build assets keep their cache policy", async (t) => { + const cases = [ + ["pagefind/pagefind.js", "text/javascript; charset=utf-8", mutableBrowser, mutableEdge], + ["pagefind/pagefind-worker.js", "text/javascript; charset=utf-8", mutableBrowser, mutableEdge], + ["pagefind/v1.6.0/pagefind.js", "text/javascript; charset=utf-8", immutable, immutable], + ["pagefind/v1.6.0/pagefind-worker.js", "text/javascript; charset=utf-8", immutable, immutable], + ["pagefind/v1.6.0/pagefind-entry.json", "application/json; charset=utf-8", mutableBrowser, mutableEdge], + ["pagefind/v1.6.0/pagefind.en.pf_meta", "application/octet-stream", immutable, immutable], + ]; + const p = harness(t, Object.fromEntries(cases.map(([key, type]) => [key, object("fixture", type)]))); + for (const [key, , browser, edge] of cases) { + policy(await p.request(`/${key}`), browser, edge); + } +}); + for (const [key, type, body, browser, edge] of assets) { for (const encoded of [false, true]) { test(`static companion cannot negotiate: ${key}, encoded=${encoded}`, async (t) => { diff --git a/workers/docs-router.ts b/workers/docs-router.ts index 54a9c321bb..42fb7cef42 100644 --- a/workers/docs-router.ts +++ b/workers/docs-router.ts @@ -600,6 +600,9 @@ function edgeCacheControlFor(pathname: string, isHtml: boolean): string { function isMutableStaticPath(pathname: string): boolean { return isShellAssetPath(pathname) + // Older Pagefind builds reused these names across incompatible versions. + || pathname === "/pagefind/pagefind.js" + || pathname === "/pagefind/pagefind-worker.js" || pathname.endsWith(".md") || pathname.endsWith(".txt") || pathname.endsWith(".xml")