From e3f00e45872e2d614bca19ee1464d1c57a0bf0c4 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:36:43 -0600 Subject: [PATCH] fix: discourage crawlers from following Discord invites --- scripts/docs-site/build.mjs | 38 ++++++++++++------- scripts/docs-site/community-invite.test.mjs | 14 +++++++ .../docs-site/discord-article-links.test.mjs | 31 ++++++++++++--- 3 files changed, 65 insertions(+), 18 deletions(-) diff --git a/scripts/docs-site/build.mjs b/scripts/docs-site/build.mjs index 588b4f6fc5..4b20ea234c 100644 --- a/scripts/docs-site/build.mjs +++ b/scripts/docs-site/build.mjs @@ -329,22 +329,34 @@ function rewriteOfficialDiscordLinks(html, slug) { if (name === "a" && attributes.href && !ancestors.some(tag => ["pre", "code", "script", "style"].includes(tag))) { let url; try { url = new URL(attributes.href); } catch {} - // Queries/fragments may carry event-specific meaning. Keep those, other - // communities, service URLs, and credential-bearing URLs untouched. - if (url && ["http:", "https:"].includes(url.protocol) && !url.username && !url.password - && !url.port && !url.search && !url.hash + const publicUrl = url && ["http:", "https:"].includes(url.protocol) && !url.username && !url.password && !url.port; + // Direct queries/fragments may carry event-specific meaning. Keep those, + // other communities and service URLs untouched; preserve tagged redirects. + const directInvite = publicUrl && !url.search && !url.hash && ((url.hostname === "discord.gg" && /^\/clawd\/?$/.test(url.pathname)) - || (["discord.com", "discordapp.com"].includes(url.hostname) && /^\/invite\/clawd\/?$/.test(url.pathname)))) { + || (["discord.com", "discordapp.com"].includes(url.hostname) && /^\/invite\/clawd\/?$/.test(url.pathname))); + const taggedInvite = publicUrl && url.hostname === "openclaw.ai" && /^\/discord\/?$/.test(url.pathname); + if (directInvite || taggedInvite) { const start = parser.startIndex; const end = parser.endIndex + 1; const opening = html.slice(start, end); - let replaced = false; - const replacement = opening.replace(/(\s+)([^\s"'<>/=]+)(\s*=\s*)(?:"[^"]*"|'[^']*'|[^\s>]+)/g, + let replacedHref = false; + let replacedRel = false; + const hasNofollow = (attributes.rel ?? "").split(/\s+/).some(token => token.toLowerCase() === "nofollow"); + const rel = [attributes.rel, "nofollow"].filter(Boolean).join(" "); + let replacement = opening.replace(/(\s+)([^\s"'<>/=]+)(?:(\s*=\s*)(?:"[^"]*"|'[^']*'|[^\s>]+))?/g, (attribute, space, name, equals) => { - if (replaced || name.toLowerCase() !== "href") return attribute; - replaced = true; - return `${space}${name}${equals}"https://openclaw.ai/discord?from=${placement}"`; + if (directInvite && !replacedHref && name.toLowerCase() === "href") { + replacedHref = true; + return `${space}${name}${equals}"https://openclaw.ai/discord?from=${placement}"`; + } + if (!hasNofollow && !replacedRel && name.toLowerCase() === "rel") { + replacedRel = true; + return `${space}${name}="${escapeAttr(rel)}"`; + } + return attribute; }); + if (!hasNofollow && !replacedRel) replacement = replacement.replace(/(\s*\/?>)$/, ` rel="nofollow"$1`); edits.push({ start, end, replacement }); } } @@ -479,7 +491,7 @@ function siteFooter(analyticsEligible = false) {

An OpenClaw Foundation project.

- +
${groups.map(([title, links]) => ``).join("")} @@ -522,7 +534,7 @@ function communityInvite(locale) {

${escapeHtml(strings.communityBody)}

${icon("reddit")}${escapeHtml(strings.communityJoin)} -${icon("discord")}${escapeHtml(strings.communityJoin)} +${icon("discord")}${escapeHtml(strings.communityJoin)} ${icon("x-social")}${escapeHtml(strings.communityFollow)}
@@ -557,7 +569,7 @@ function socialIconLinks(placement) { ["Reddit", "https://www.reddit.com/r/openclaw/", "reddit"], ["Discord", `https://openclaw.ai/discord?from=${placement}`, "discord"], ["X", "https://x.com/openclaw", "x-social"], - ].map(([label, href, iconName]) => `${icon(iconName)}`).join(""); + ].map(([label, href, iconName]) => `${icon(iconName)}`).join(""); } function firstStatusLine(content) { diff --git a/scripts/docs-site/community-invite.test.mjs b/scripts/docs-site/community-invite.test.mjs index 0886c06f22..071c52ca9f 100644 --- a/scripts/docs-site/community-invite.test.mjs +++ b/scripts/docs-site/community-invite.test.mjs @@ -50,6 +50,20 @@ test("community artwork opts out of Pinterest without changing the invitation or [strings.communityJoin, "https://openclaw.ai/discord?from=docs-community-invite"], [strings.communityFollow, "https://x.com/openclaw"], ]); + const anchors = all(node => node.name === "a"); + for (const [tag, rel] of [ + ["docs-header", "nofollow"], ["docs-sidebar", "nofollow"], + ["docs-footer", "noopener nofollow"], ["docs-community-invite", "noopener noreferrer nofollow"], + ...(slug === "index" ? [["docs-home-community", "nofollow"]] : []), + ]) { + const link = anchors.find(node => node.attribs.href === `https://openclaw.ai/discord?from=${tag}`); + assert.equal(link?.attribs.rel, rel, `${locale}/${slug}: ${tag}`); + } + for (const link of anchors.filter(node => !node.attribs.href?.startsWith("https://openclaw.ai/discord?"))) { + assert.ok(!link.attribs.rel?.split(/\s+/).includes("nofollow"), `${locale}/${slug}: non-invite rel unchanged`); + } + assert.equal(links[0].attribs.rel, "noopener noreferrer"); + assert.equal(links[2].attribs.rel, "noopener noreferrer"); const optedOut = all(node => node.attribs && ("nopin" in node.attribs || "data-pin-nopin" in node.attribs)); assert.deepEqual(optedOut, art, `${locale}/${slug}: only invitation images opt out`); assert.equal(all(node => node.name === "meta" && node.attribs.name === "pinterest").length, 0); diff --git a/scripts/docs-site/discord-article-links.test.mjs b/scripts/docs-site/discord-article-links.test.mjs index bf4151a406..442ac1de4a 100644 --- a/scripts/docs-site/discord-article-links.test.mjs +++ b/scripts/docs-site/discord-article-links.test.mjs @@ -18,16 +18,20 @@ const preserved = [ "https://discord.com/api/v10/invites/clawd", "https://discord.gg/clawd?event=123456789012345678", "https://discord.com/invite/clawd#event", "https://discord.gg/clawd?utm_source=legacy", "https://discord.gg.example.org/clawd", "https://example.org/?invite=https://discord.gg/clawd", - "https://openclaw.ai/discord?from=docs-plugins-building-plugins", "https://community.openclaw.ai/", + "https://openclaw.ai/discord-extra?from=docs", "https://openclaw.ai.example.org/discord?from=docs", "https://community.openclaw.ai/", ]; +const tagged = ["https://openclaw.ai/discord?from=docs-plugins-building-plugins", "https://openclaw.ai/discord/?from=docs&keep=value#join"]; function article(document) { return DomUtils.findOne(node => node.attribs?.class?.split(" ").includes("doc"), document.children); } -test("English and localized rendered article anchors use canonical page placements, including cached articles", t => { +test("English and localized invite anchors retain canonical placements and add nofollow, including cached articles", t => { const body = ["# Community", ...official.map((href, i) => `[Official ${i}](${href})`), - 'Raw HTML', + 'Raw HTML', + 'Already nofollow', + 'Empty rel', + ...tagged.map((href, i) => `[Tagged ${i}](${href})`), ...preserved.map((href, i) => `[Preserve ${i}](${href})`), "`https://discord.gg/clawd`", "```html\nExample\n```", '

https://discord.gg/clawd

', 'Code anchor', @@ -45,15 +49,32 @@ test("English and localized rendered article anchors use canonical page placemen const doc = article(parseDocument(html)); assert.ok(doc, `${locale}/${slug}`); const anchors = DomUtils.findAll(node => node.name === "a", doc.children); - for (let i = 0; i < official.length; i++) assert.equal(anchors.find(node => DomUtils.textContent(node) === `Official ${i}`).attribs.href, `https://openclaw.ai/discord?from=${tag}`); + for (let i = 0; i < official.length; i++) { + const link = anchors.find(node => DomUtils.textContent(node) === `Official ${i}`); + assert.equal(link.attribs.href, `https://openclaw.ai/discord?from=${tag}`); + assert.equal(link.attribs.rel, "nofollow"); + } const raw = anchors.find(node => node.attribs.class === "raw-link"); assert.equal(raw.attribs.href, `https://openclaw.ai/discord?from=${tag}`); + assert.equal(raw.attribs.rel, "noopener noreferrer nofollow"); assert.equal(raw.attribs["data-href"], "https://example.org/unchanged"); assert.equal(raw.attribs.title, "Keep href=example title"); - for (let i = 0; i < preserved.length; i++) assert.equal(anchors.find(node => DomUtils.textContent(node) === `Preserve ${i}`).attribs.href, preserved[i]); + assert.equal(anchors.find(node => DomUtils.textContent(node) === "Already nofollow").attribs.rel, "ugc NOFOLLOW"); + assert.equal(anchors.find(node => DomUtils.textContent(node) === "Empty rel").attribs.rel, "nofollow"); + for (let i = 0; i < tagged.length; i++) { + const link = anchors.find(node => DomUtils.textContent(node) === `Tagged ${i}`); + assert.equal(link.attribs.href, tagged[i]); + assert.equal(link.attribs.rel, "nofollow"); + } + for (let i = 0; i < preserved.length; i++) { + const link = anchors.find(node => DomUtils.textContent(node) === `Preserve ${i}`); + assert.equal(link.attribs.href, preserved[i]); + assert.equal(link.attribs.rel, undefined); + } const code = DomUtils.findAll(node => node.name === "code", doc.children).map(node => DomUtils.textContent(node)); assert.ok(code.some(text => text.includes('href="https://discord.gg/clawd"')), "fenced code is unchanged"); assert.ok(code.some(text => text === "https://discord.gg/clawd"), "inline code is unchanged"); assert.equal(anchors.find(node => DomUtils.textContent(node) === "Code anchor").attribs.href, "https://discord.gg/clawd"); + assert.equal(anchors.find(node => DomUtils.textContent(node) === "Code anchor").attribs.rel, undefined); const display = DomUtils.findOne(node => node.attribs?.class === "display-url", doc.children); assert.equal(DomUtils.textContent(display), "https://discord.gg/clawd", "display text is not rewritten"); assert.equal(fs.readFileSync(path.join(f.root, "docs", locale === "en" ? "" : locale, slug + ".md"), "utf8"), body);