diff --git a/.github/workflows/build_and_check.yml b/.github/workflows/build_and_check.yml index d9d2c83fe..dea9c9785 100644 --- a/.github/workflows/build_and_check.yml +++ b/.github/workflows/build_and_check.yml @@ -21,6 +21,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + # Full history, so the preview's last-update dates and sitemap + # match what production will ship. + fetch-depth: 0 - name: Set up Node.js uses: actions/setup-node@v5 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a6b13c19f..3cf12a332 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,6 +47,9 @@ jobs: with: # main as it is now, not github.sha — see the concurrency comment. ref: main + # Full history: doc last-update dates and sitemap come + # from each file's git log (src/lastUpdate/contentLastUpdate.js). + fetch-depth: 0 - name: Set Node.js 22.x uses: actions/setup-node@v3 diff --git a/.lastmod-ignore-revs b/.lastmod-ignore-revs new file mode 100644 index 000000000..3bbd79084 --- /dev/null +++ b/.lastmod-ignore-revs @@ -0,0 +1,5 @@ +# Commits that change doc bodies without changing what they say (a mass +# find-and-replace, a link or formatting rewrite). Doc last-update dates and +# the sitemap skip them. Front-matter-only and whitespace-only +# changes are already skipped automatically, so they do not need listing. +# One full commit hash per line; see src/lastUpdate/contentLastUpdate.js. diff --git a/docusaurus.config.js b/docusaurus.config.js index 59ad65d9b..fa11adc6b 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -8,6 +8,7 @@ const fs = require("fs"); const remarkImageSize = require("./src/remark/remarkImageSize"); import {visit} from "unist-util-visit"; const FontPreloadPlugin = require("webpack-font-preload-plugin"); +const {withContentLastUpdate} = require("./src/lastUpdate/contentLastUpdate"); // Single source of truth for the "current" docs version. Update this one // value when cutting a new docs version -- it drives both the versioning @@ -170,6 +171,17 @@ s.parentNode.insertBefore(b, s);})(window.lintrk);`, baseUrl: "/docs/", onBrokenLinks: "throw", trailingSlash: true, + markdown: { + // Date each doc by the last commit that changed its content, not the last + // commit that touched the file (version cuts, front-matter sweeps). Feeds + // the "Last updated" line, TechArticle dateModified and sitemap . + // See src/lastUpdate/contentLastUpdate.js. + parseFrontMatter: async (params) => + withContentLastUpdate( + params.filePath, + await params.defaultParseFrontMatter(params) + ), + }, favicon: "img/favicon.png", organizationName: "keploy", // Usually your GitHub org/user name. projectName: "docs", // Usually your repo name. @@ -509,6 +521,10 @@ s.parentNode.insertBefore(b, s);})(window.lintrk);`, //cacheTime: 600 * 1000, // 600 sec - cache purge period changefreq: "weekly", priority: 0.5, + // from each doc's last content change (date only, which + // is all the precision Google uses). Only accurate with full git + // history: CI checks out with fetch-depth: 0. + lastmod: "date", filename: "sitemap.xml", // Differentiate docs sitemap priorities by content type so // search engines spend crawl budget proportional to how diff --git a/src/lastUpdate/contentLastUpdate.js b/src/lastUpdate/contentLastUpdate.js new file mode 100644 index 000000000..efe02c09d --- /dev/null +++ b/src/lastUpdate/contentLastUpdate.js @@ -0,0 +1,223 @@ +// contentLastUpdate — the date a doc's *content* last changed, for the +// "Last updated" line, the TechArticle `dateModified`, and the sitemap +// ``. +// +// Why not Docusaurus's own git date: it takes the newest commit that touched +// the file, whatever that commit did. In this repo that is badly wrong: +// - cutting a docs version copies every page into versioned_docs/version-X/, +// so every page "changes" on the day the version is cut; +// - sitewide sweeps (front matter / SEO fields, formatting) touch most pages +// without changing what they say. +// A that moves when nothing a reader sees has changed is one Google +// learns to ignore, for the whole sitemap. +// +// What counts as a content change: the Markdown body (everything after the +// front matter) differs once whitespace is collapsed. Walking a file's history +// newest-first, commits that only touch front matter or whitespace are +// skipped, and when the file was created by a copy or rename (a version cut, a +// move) the walk carries on through the file it came from, as long as the body +// came across unchanged. +// +// Commits that change bodies without changing content (a mass find-and-replace, +// a link rewrite) can be skipped by listing them in .lastmod-ignore-revs at the +// repo root, the same way .git-blame-ignore-revs works for blame. +// +// The result is injected as `last_update` front matter (see parseFrontMatter in +// docusaurus.config.js), which Docusaurus already prefers over git. A +// `last_update.date` written by hand in a doc always wins. +// +// Safety: this never throws. When it cannot answer (dev server, no git, a +// shallow clone, a file git does not track) it returns null and Docusaurus +// falls back to its own behaviour. + +const {execFile} = require("child_process"); +const fs = require("fs"); +const path = require("path"); + +const REPO_ROOT = path.resolve(__dirname, "..", ".."); +const IGNORE_REVS_FILE = path.join(REPO_ROOT, ".lastmod-ignore-revs"); + +// Only docs content is dated this way; pages and partials keep the default. +const DOC_DIRS = ["docs", "versioned_docs"].map( + (dir) => path.join(REPO_ROOT, dir) + path.sep +); + +// ~400 docs are parsed concurrently at build time; cap the git processes. +const MAX_GIT_PROCESSES = 16; +let runningGit = 0; +const gitQueue = []; + +function git(args) { + return new Promise((resolve) => { + const start = () => { + runningGit += 1; + execFile( + "git", + args, + {cwd: REPO_ROOT, maxBuffer: 64 * 1024 * 1024}, + (err, stdout) => { + runningGit -= 1; + if (gitQueue.length) gitQueue.shift()(); + resolve(err ? null : stdout); + } + ); + }; + if (runningGit < MAX_GIT_PROCESSES) start(); + else gitQueue.push(start); + }); +} + +function memoize(fn) { + const cache = new Map(); + return (...args) => { + const key = args.join("\0"); + if (!cache.has(key)) cache.set(key, fn(...args)); + return cache.get(key); + }; +} + +function readIgnoreRevs() { + try { + return fs + .readFileSync(IGNORE_REVS_FILE, "utf8") + .split("\n") + .map((line) => line.replace(/#.*/, "").trim()) + .filter(Boolean); + } catch { + return []; + } +} +const ignoreRevs = readIgnoreRevs(); +const isIgnored = (hash) => ignoreRevs.some((rev) => hash.startsWith(rev)); + +// Body with front matter removed and whitespace collapsed, so front-matter +// edits, re-wrapping and trailing-space fixes all compare equal. +function contentOf(source) { + if (source === null) return null; + return source + .replace(/^?---\r?\n[\s\S]*?\r?\n---(\r?\n|$)/, "") + .replace(/\s+/g, " ") + .trim(); +} + +const readContent = memoize(async (rev, file) => + contentOf(await git(["show", `${rev}:${file}`])) +); + +// For a commit, which added paths were copied or renamed, and from where. +const copySources = memoize(async (hash) => { + const out = await git([ + "diff", + "--name-status", + "-M", + "-C", + "--find-copies-harder", + "-l0", + `${hash}^`, + hash, + ]); + const sources = new Map(); + for (const line of (out || "").split("\n")) { + const [status, from, to] = line.split("\t"); + if (to && /^[RC]/.test(status)) sources.set(to, from); + } + return sources; +}); + +// Newest commit, at or before `rev`, that changed the content of `file`. +async function lastContentChange(file, rev) { + const log = await git([ + "log", + "--no-merges", + "--format=%H%x09%ct%x09%an", + rev, + "--", + file, + ]); + if (!log) return null; + + for (const line of log.trim().split("\n")) { + const [hash, timestamp, author] = line.split("\t"); + const change = {timestamp: Number(timestamp) * 1000, author}; + const ignored = isIgnored(hash); + const after = await readContent(hash, file); + if (after === null) continue; // the commit deleted the file + const before = await readContent(`${hash}^`, file); + + if (before === null) { + // The file was created here. If it is a copy or rename, keep walking + // through the original, unless the content changed on the way over. + const source = (await copySources(hash)).get(file); + if (!source) return ignored ? null : change; + const sourceContent = await readContent(`${hash}^`, source); + if (!ignored && sourceContent !== after) return change; + return lastContentChange(source, `${hash}^`); + } + + if (!ignored && before !== after) return change; + } + return null; +} + +let enabled; +function isEnabled() { + if (enabled !== undefined) return enabled; + // Docusaurus itself skips git in dev and when this env var is set; match it. + if ( + process.env.NODE_ENV !== "production" || + process.env.DOCUSAURUS_DISABLE_LAST_UPDATE === "true" + ) { + return (enabled = false); + } + const shallow = require("child_process") + .spawnSync("git", ["rev-parse", "--is-shallow-repository"], { + cwd: REPO_ROOT, + encoding: "utf8", + }) + .stdout?.trim(); + if (shallow !== "false") { + // A shallow clone has no history to walk; every page would get the date + // of the newest commit. CI must check out with fetch-depth: 0. + console.warn( + "[contentLastUpdate] git history unavailable (shallow clone?): " + + "doc last-update dates and sitemap will not be accurate." + ); + return (enabled = false); + } + return (enabled = true); +} + +const lastUpdateFor = memoize(async (filePath) => { + const file = path.relative(REPO_ROOT, filePath).split(path.sep).join("/"); + try { + return await lastContentChange(file, "HEAD"); + } catch { + return null; + } +}); + +// Adds `last_update` to a parsed doc's front matter. Hand-written values win. +async function withContentLastUpdate(filePath, parsed) { + const {frontMatter} = parsed; + if ( + !isEnabled() || + frontMatter.last_update?.date || + !DOC_DIRS.some((dir) => filePath.startsWith(dir)) + ) { + return parsed; + } + const change = await lastUpdateFor(filePath); + if (!change) return parsed; + return { + ...parsed, + frontMatter: { + ...frontMatter, + last_update: { + date: new Date(change.timestamp).toISOString(), + author: frontMatter.last_update?.author ?? change.author, + }, + }, + }; +} + +module.exports = {withContentLastUpdate, lastUpdateFor};