Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"private": true,
"type": "module",
"scripts": {
"docs:build": "node scripts/docs-site/build.mjs && node scripts/docs-site/search-index.mjs && node scripts/docs-site/source-index.mjs && pagefind --site dist/docs-site --output-path dist/docs-site/pagefind && node scripts/docs-site/pagefind-normalize.mjs",
"docs:build": "node scripts/docs-site/build.mjs && node scripts/docs-site/search-index.mjs && node scripts/docs-site/source-index.mjs && node scripts/docs-site/pagefind-build.mjs && node scripts/docs-site/pagefind-normalize.mjs",
"docs:build:shell": "DOCS_SITE_ARTIFACT_MODE=shell node scripts/docs-site/build.mjs",
"docs:build:preview": "DOCS_SITE_ARTIFACT_MODE=shell DOCS_SITE_PREVIEW_LOCALE=en DOCS_SITE_PREVIEW_PAGES_PER_GROUP=2 DOCS_SITE_PREVIEW_MAX_PAGES=30 node scripts/docs-site/build.mjs",
"docs:build:r2": "npm run docs:build && node scripts/docs-site/r2-prepare.mjs",
Expand Down
3 changes: 2 additions & 1 deletion scripts/docs-site/analytics-behavior.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import path from "node:path";
import { before, after, test } from "node:test";
import { chromium } from "playwright";
import { fixture, write } from "./test-helpers/redirect-fixture.mjs";
import { pagefindAssetDir } from "./pagefind-assets.mjs";

const origin = "https://docs.openclaw.ai";
const sdk = process.env.DOCS_GA4_SDK_PATH ? fs.readFileSync(process.env.DOCS_GA4_SDK_PATH, "utf8") : "";
Expand Down Expand Up @@ -43,7 +44,7 @@ async function openSite(t, { privacy, clock = false, diagram = false, articleLin
if (url.pathname === "/ask-molty/api/chat") return route.fulfill(request.method() === "GET"
? { contentType: "application/json", body: "{}" }
: { contentType: "text/plain", body: "[Private reference](https://private.example/PRIVATE_CHAT_VALUE?q=PRIVATE_CHAT_QUERY)" });
if (url.pathname === "/pagefind/pagefind.js") return route.fulfill({ contentType: "text/javascript", body: `
if (url.pathname === `/${pagefindAssetDir}/pagefind.js`) return route.fulfill({ contentType: "text/javascript", body: `
export async function init() {}
export async function search(q) { if(q.includes("delay")) await window.__searchWait; if(q.includes("error")) throw new Error("Fixture search error"); return {results: Array.from({length:q.includes("missing")?0:q.includes("many")?20:1},(_,i)=>({data: async () => ({url:"/guide#result-"+i,meta:{title:"Guide"},excerpt:"Public guidance"})}))}; }
` });
Expand Down
10 changes: 10 additions & 0 deletions scripts/docs-site/pagefind-assets.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";

const packageRoot = path.dirname(path.dirname(fileURLToPath(import.meta.resolve("pagefind"))));
export const pagefindVersion = JSON.parse(fs.readFileSync(path.join(packageRoot, "package.json"), "utf8")).version;
if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(pagefindVersion)) {
throw new Error(`Invalid installed Pagefind version: ${pagefindVersion}`);
}
export const pagefindAssetDir = `pagefind/v${pagefindVersion}`;
16 changes: 16 additions & 0 deletions scripts/docs-site/pagefind-build.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#!/usr/bin/env node
import { spawnSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { pagefindAssetDir } from "./pagefind-assets.mjs";

const site = path.join(process.cwd(), "dist/docs-site");
if (!fs.existsSync(site)) throw new Error("dist/docs-site does not exist; run build.mjs first");

const runner = fileURLToPath(new URL("../../node_modules/pagefind/lib/runner/bin.cjs", import.meta.url));
const result = spawnSync(process.execPath, [runner, "--site", site, "--output-path", path.join(site, pagefindAssetDir)], {
stdio: "inherit",
});
if (result.error) throw result.error;
if (result.status !== 0) process.exit(result.status ?? 1);
98 changes: 98 additions & 0 deletions scripts/docs-site/pagefind-cache.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
import assert from "node:assert/strict";
import fs from "node:fs";
import http from "node:http";
import path from "node:path";
import { test } from "node:test";
import { chromium } from "playwright";
import { fixture, repo, run, write } from "./test-helpers/redirect-fixture.mjs";
import { pagefindAssetDir, pagefindVersion } from "./pagefind-assets.mjs";

test("Pagefind search uses the current build after immutable entry and worker assets were cached", async (t) => {
const f = fixture(t, [], {
"cron.md": "# Cron scheduler\n\nUse the cron scheduler to run recurring jobs.\n",
});
write(f.root, "docs/docs.json", JSON.stringify({
name: "Search fixture",
navigation: { languages: [{ language: "en", tabs: [{ tab: "Docs", groups: [{ group: "Guide", pages: ["index", "cron"] }] }] }] },
}));
fs.symlinkSync(path.join(repo, "node_modules"), path.join(f.root, "node_modules"), process.platform === "win32" ? "junction" : "dir");
const built = f.build();
assert.equal(built.status, 0, built.stderr);
const site = path.join(f.root, "dist/docs-site");
const pagefind = run(f.root, "pagefind-build.mjs");
assert.equal(pagefind.status, 0, pagefind.stderr);
const normalized = run(f.root, "pagefind-normalize.mjs");
assert.equal(normalized.status, 0, normalized.stderr);
assert.equal(JSON.parse(fs.readFileSync(path.join(site, pagefindAssetDir, "pagefind-entry.json"), "utf8")).version, pagefindVersion);
assert.ok(fs.readFileSync(path.join(site, "assets/docs-site.js"), "utf8").includes(`/${pagefindAssetDir}/pagefind.js`));
for (const name of ["pagefind.js", "pagefind-worker.js"]) {
assert.ok(fs.readFileSync(path.join(site, pagefindAssetDir, name), "utf8").includes(`pagefind_version="${pagefindVersion}"`),
`${name} must match the generated index version`);
}

// Model the reported failure boundary: the old entry imports and initializes,
// then its search rejects after reading the newer index, so import retry does
// not run. The worker is the real 1.5.2 output from a one-page index build.
const legacyEntry = 'export async function init() {}\nexport async function search() { throw new Error("Pagefind WASM Error (No pointer): 1.5.2 runtime, 1.6.0 index"); }\n';
const legacyWorker = fs.readFileSync(new URL("./test-fixtures/pagefind-1.5.2/pagefind-worker.js", import.meta.url));
let currentBuild = true;
const legacyCached = new Set();
const currentRequests = [];
const server = http.createServer((req, res) => {
const url = new URL(req.url, "http://localhost");
if (url.pathname === "/warm") return res.setHeader("Content-Type", "text/html").end("<!doctype html><title>Previous visit</title>");
const legacyPath = url.pathname === "/pagefind/pagefind.js" || url.pathname === "/pagefind/pagefind-worker.js";
const cacheKey = url.pathname + url.search;
if (legacyPath && (!currentBuild || legacyCached.has(cacheKey))) {
if (!currentBuild) legacyCached.add(cacheKey);
res.setHeader("Content-Type", "text/javascript");
res.setHeader("Cache-Control", "public, max-age=31536000, immutable");
return res.end(url.pathname.endsWith("-worker.js") ? legacyWorker : legacyEntry);
}
if (currentBuild) currentRequests.push(url.pathname);
const file = path.join(site, decodeURIComponent(url.pathname).replace(/^\/+/, ""));
if (!fs.existsSync(file)) return res.writeHead(404).end();
const resolved = fs.statSync(file).isDirectory() ? path.join(file, "index.html") : file;
if (!fs.existsSync(resolved)) return res.writeHead(404).end();
const contentType = ({ ".html": "text/html", ".js": "text/javascript", ".css": "text/css", ".json": "application/json", ".wasm": "application/wasm" })[path.extname(resolved)] || "application/octet-stream";
res.setHeader("Content-Type", contentType);
res.setHeader("Cache-Control", "public, max-age=31536000, immutable");
res.end(fs.readFileSync(resolved));
});
await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));
t.after(() => new Promise(resolve => server.close(resolve)));
const browser = await chromium.launch({ headless: true });
t.after(() => browser.close());
const base = `http://127.0.0.1:${server.address().port}`;

async function search(page) {
await page.goto(base + "/", { waitUntil: "domcontentloaded" });
await page.locator("[data-search-open]").first().click();
await page.locator("[data-search-input]").fill("cron scheduler");
await page.locator(".search-result, .search-unavailable").first().waitFor({ timeout: 10000 });
assert.match(await page.locator("[data-search-results]").innerText(), /Cron scheduler/i);
}

const cold = await browser.newContext();
t.after(() => cold.close());
await search(await cold.newPage());
currentRequests.length = 0;
currentBuild = false;

const warm = await browser.newContext();
t.after(() => warm.close());
const warmPage = await warm.newPage();
await warmPage.goto(base + "/warm");
const workerStarted = warmPage.waitForEvent("worker");
await warmPage.evaluate(async () => {
await import("/pagefind/pagefind.js");
window.__legacyWorker = new Worker("/pagefind/pagefind-worker.js", { type: "module" });
});
await workerStarted;
await warmPage.evaluate(() => window.__legacyWorker.terminate());
assert.deepEqual([...legacyCached].sort(), ["/pagefind/pagefind-worker.js", "/pagefind/pagefind.js"]);
currentBuild = true;
await search(warmPage);
assert.ok(currentRequests.some(url => /pagefind-worker\.js$/.test(url)),
"the upgraded search must request a worker outside the old immutable cache key");
});
6 changes: 5 additions & 1 deletion scripts/docs-site/pagefind-normalize.mjs
Original file line number Diff line number Diff line change
@@ -1,13 +1,17 @@
#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";
import { pagefindAssetDir, pagefindVersion } from "./pagefind-assets.mjs";

const root = process.cwd();
const entryPath = path.join(root, "dist", "docs-site", "pagefind", "pagefind-entry.json");
const entryPath = path.join(root, "dist", "docs-site", pagefindAssetDir, "pagefind-entry.json");

if (!fs.existsSync(entryPath)) throw new Error("pagefind-entry.json does not exist; run pagefind first");

const entry = JSON.parse(fs.readFileSync(entryPath, "utf8"));
if (entry.version !== pagefindVersion) {
throw new Error(`Pagefind index version ${entry.version} does not match installed runtime ${pagefindVersion}`);
}
if (entry.languages && typeof entry.languages === "object" && !Array.isArray(entry.languages)) {
entry.languages = Object.fromEntries(
Object.entries(entry.languages).sort(([left], [right]) => left.localeCompare(right)),
Expand Down
9 changes: 2 additions & 7 deletions scripts/docs-site/publication-stability.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@ import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { spawnSync } from "node:child_process";
import test from "node:test";
import { fixture, repo, run, write } from "./test-helpers/redirect-fixture.mjs";
import { fixture, run, write } from "./test-helpers/redirect-fixture.mjs";
import { siteJs } from "./site-js.mjs";

test("runtime identity hashes the executed source, including analytics and lazy dependencies", () => {
Expand All @@ -31,11 +30,7 @@ test("commit-only builds preserve artifacts; two body edits change only their pa
const result = run(f.root, script, env);
assert.equal(result.status, 0, result.stderr);
}
const pagefind = spawnSync(path.join(repo, "node_modules/.bin/pagefind"), ["--site", site], {
env: { PATH: process.env.PATH }, encoding: "utf8",
});
assert.equal(pagefind.status, 0, pagefind.stderr);
for (const script of ["pagefind-normalize.mjs", "r2-prepare.mjs"]) {
for (const script of ["pagefind-build.mjs", "pagefind-normalize.mjs", "r2-prepare.mjs"]) {
const result = run(f.root, script, env);
assert.equal(result.status, 0, result.stderr);
}
Expand Down
3 changes: 2 additions & 1 deletion scripts/docs-site/r2-prepare.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,8 @@ function cacheControlFor(key) {
if (key.endsWith(".html") || !path.extname(key)) {
return "public, max-age=60, s-maxage=86400, stale-while-revalidate=604800";
}
if (key.endsWith(".md") || key.endsWith(".txt") || key.endsWith(".json") || key.endsWith(".jsonl")) {
if (key === "pagefind/pagefind.js" || key === "pagefind/pagefind-worker.js"
|| key.endsWith(".md") || key.endsWith(".txt") || key.endsWith(".json") || key.endsWith(".jsonl")) {
return "public, max-age=300, s-maxage=3600, stale-while-revalidate=86400";
}
return "public, max-age=31536000, immutable";
Expand Down
17 changes: 17 additions & 0 deletions scripts/docs-site/r2-upload.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,23 @@ test("prepared artifact paths preserve uploaded bytes and reuse old manifest obj
assert.deepEqual(puts(cold.stdout), prepared.entries.map((entry) => entry.key));
});

test("R2 Pagefind metadata distinguishes legacy scripts from versioned build assets", (t) => {
const f = fixture(t);
const expected = new Map([
["pagefind/pagefind.js", "public, max-age=300, s-maxage=3600, stale-while-revalidate=86400"],
["pagefind/pagefind-worker.js", "public, max-age=300, s-maxage=3600, stale-while-revalidate=86400"],
["pagefind/v1.6.0/pagefind.js", "public, max-age=31536000, immutable"],
["pagefind/v1.6.0/pagefind-worker.js", "public, max-age=31536000, immutable"],
["pagefind/v1.6.0/pagefind-entry.json", "public, max-age=300, s-maxage=3600, stale-while-revalidate=86400"],
["pagefind/v1.6.0/pagefind.en.pf_meta", "public, max-age=31536000, immutable"],
]);
for (const key of expected.keys()) write(f.root, `dist/docs-site/${key}`, "fixture");
const entries = new Map(f.prepare().entries.map((entry) => [entry.key, entry]));
for (const [key, cacheControl] of expected) {
assert.equal(entries.get(key)?.cacheControl, cacheControl, key);
}
});

for (const mode of ["manifest", "head", "head-error"]) {
test(`${mode} audit uploads metadata add/change/remove despite unchanged ETags using only mocked fetch`, (t) => {
const f = uploadFixture(t);
Expand Down
3 changes: 2 additions & 1 deletion scripts/docs-site/site-js.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { createDocsAnalytics } from "./analytics.mjs";
import { createDocsAnalyticsEvents } from "./analytics-events.mjs";
import { webVitalsAssetName } from "./web-vitals-runtime.mjs";
import { peleAssetName } from "./pele-runtime.mjs";
import { pagefindAssetDir } from "./pagefind-assets.mjs";
import { initDocsAnalyticsConsent } from "./analytics-consent.mjs";
// The emitted runtime also supports the classic-script navigation harness.
const heroRuntime = fs.readFileSync(new URL("./hero-art.mjs", import.meta.url), "utf8").replace(/^export /gm, "");
Expand Down Expand Up @@ -279,7 +280,7 @@ function currentTocHeadingId(headings){const top=parseFloat(getComputedStyle(doc
function initTocScrollspy(){tocObserver?.disconnect();tocObserver=null;tocSizeObserver?.disconnect();tocSizeObserver=null;if(tocScrollHandler){removeEventListener("scroll",tocScrollHandler);tocScrollHandler=null;}const toc=document.querySelector(".toc");if(!toc)return;if("ResizeObserver" in window){tocSizeObserver=new ResizeObserver(syncTocSpace);tocSizeObserver.observe(toc);const nav=toc.querySelector("nav");if(nav)tocSizeObserver.observe(nav)}const links=[...toc.querySelectorAll('a[href^="#"]')];const headings=[...document.querySelectorAll(".doc h2[id],.doc h3[id]")].filter(heading=>links.some(link=>tocLinkId(link)===heading.id));links.forEach(link=>link.classList.remove("active"));if(!headings.length)return;const sync=()=>{if(Date.now()<tocSpyHoldUntil)return;setActiveTocLink(currentTocHeadingId(headings))};let tocScrollRaf=0;tocScrollHandler=()=>{cancelAnimationFrame(tocScrollRaf);tocScrollRaf=requestAnimationFrame(sync)};addEventListener("scroll",tocScrollHandler,{passive:true});requestAnimationFrame(sync);if(!("IntersectionObserver" in window)){sync();return}tocObserver=new IntersectionObserver(sync,{rootMargin:"-120px 0px -70% 0px",threshold:[0,1]});headings.forEach(heading=>tocObserver.observe(heading))}
async function navigateTo(url,replace=false,animateHome=false){if(navigating)return false;navigating=true;if(!animateHome)homeGlimm.cancel();closeLanguage();closeMermaidOverlay();try{const res=await fetch(url.href,{credentials:"same-origin"});if(!res.ok||!res.headers.get("content-type")?.includes("text/html"))return false;const nextDoc=new DOMParser().parseFromString(await res.text(),"text/html");if(!nextDoc.querySelector(".main"))return false;const commit=()=>{disposeHomeHero();for(const name of ["docs-home","docs-home-layout"])document.body.classList.toggle(name,nextDoc.body.classList.contains(name));syncSidebar(nextDoc);swap(".header-left",nextDoc);swap(".main",nextDoc);initHomeHero();syncStickyHeaderOffset();syncTocDisclosure();initCodeGroups();initCodeFades();initAccordionMotion();initPageFeedback();initMermaid();document.title=nextDoc.title;history[replace?"replaceState":"pushState"]({docs:true},"",url.href);docsAnalytics.pageView();currentDocKey=location.pathname+location.search;setNavOpen(false);initTocScrollspy();scrollTarget(url.hash);syncCommunityInvite()};if(animateHome)await homeGlimm.run(commit);else commit();return true}catch{return false}finally{navigating=false}}
function unavailableSearch(){return {unavailable:true,search:async()=>({results:[]})}}
function loadPagefind(){if(pagefindReady)return pagefindReady;const src=withBase("/pagefind/pagefind.js")+(pagefindLoadAttempt?"?retry="+pagefindLoadAttempt:"");pagefindLoadAttempt+=1;pagefindReady=import(src).then(m=>m.init?.().then?.(()=>m)??m).catch(()=>{pagefindReady=null;return unavailableSearch()});return pagefindReady}
function loadPagefind(){if(pagefindReady)return pagefindReady;const src=withBase(${JSON.stringify("/" + pagefindAssetDir + "/pagefind.js")})+(pagefindLoadAttempt?"?retry="+pagefindLoadAttempt:"");pagefindLoadAttempt+=1;pagefindReady=import(src).then(m=>m.init?.().then?.(()=>m)??m).catch(()=>{pagefindReady=null;return unavailableSearch()});return pagefindReady}
let searchReturnFocus=null;
function openSearch(){if(!modal)return;closeLanguage();if(document.body.classList.contains("nav-open"))setNavOpen(false);if(!modal.classList.contains("open")){searchReturnFocus=document.activeElement;docsTelemetry.popup("docs_search",true,"","search_dialog")}modal.classList.add("open");if(!searchQuery()&&results&&!results.innerHTML.trim())results.innerHTML=searchInitialHtml();setSearchChrome(searchQuery());setTimeout(()=>input?.focus(),0);loadPagefind()}
function clearSearch(){docsTelemetry.cancelSearch();clearTimeout(searchTimer);seq+=1;searchActiveIndex=-1;setSearchLoading(false);if(input)input.value="";setSearchChrome("");if(results){results.classList.remove("is-loading");results.innerHTML=searchInitialHtml()}}
Expand Down
7 changes: 5 additions & 2 deletions scripts/docs-site/smoke.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { DomUtils, parseDocument } from "htmlparser2";
import { ignoredDocDirs, localeFlags, localeLabels, navigationLocaleToDir } from "./config.mjs";
import { chromeStrings } from "./chrome-strings.mjs";
import { peleAssetName } from "./pele-runtime.mjs";
import { pagefindAssetDir } from "./pagefind-assets.mjs";
import { homeStrings } from "./home-strings.mjs";
import { editSourceUrlForPage, frontmatterSourcePath, readSourceMetadata } from "./edit-source.mjs";
import { parseFrontmatter } from "../../.openclaw-sync/lib/docs-markdown.mjs";
Expand Down Expand Up @@ -51,7 +52,9 @@ if (!shellOnly) {
".well-known/llms.txt",
"robots.txt",
"sitemap.xml",
"pagefind/pagefind.js",
`${pagefindAssetDir}/pagefind.js`,
`${pagefindAssetDir}/pagefind-worker.js`,
`${pagefindAssetDir}/pagefind-entry.json`,
"docs-search.json",
);
}
Expand Down Expand Up @@ -291,7 +294,7 @@ if (!/class="hljs-attr">channels<\/span>/.test(index)
throw new Error("index: json5 config example was not syntax-highlighted");
}
const modelsMarkdown = fs.readFileSync(path.join(site, "concepts/models.md"), "utf8");
if (!/^---\nsummary: /m.test(modelsMarkdown) || !/title: "Models CLI"/m.test(modelsMarkdown)) {
if (!/^---\r?\nsummary: /m.test(modelsMarkdown) || !/title: "Models CLI"/m.test(modelsMarkdown)) {
throw new Error("concepts/models.md: source markdown was not emitted");
}
if (process.env.DOCS_SITE_BASE_PATH && (/src="\/assets\//.test(index) || /href="\/assets\//.test(index))) {
Expand Down
Loading