From ef0da1b692ed2c6c87d01b4f2b516b84bca7a495 Mon Sep 17 00:00:00 2001 From: Sauer Date: Tue, 20 Oct 2026 14:01:24 +0000 Subject: [PATCH] fix(web): serve /sitemap.xml with Last-Modified header (web 0.40.3) --- apps/web/app/sitemap.xml/route.test.ts | 34 ++++++++++ .../app/{sitemap.ts => sitemap.xml/route.ts} | 68 +++++++++++++++---- apps/web/lib/agent/pow.test.ts | 17 ++++- apps/web/lib/changelog.ts | 12 +++- apps/web/lib/seo.ts | 6 +- apps/web/package.json | 2 +- apps/web/scripts/seo-smoke.mjs | 6 ++ 7 files changed, 124 insertions(+), 21 deletions(-) create mode 100644 apps/web/app/sitemap.xml/route.test.ts rename apps/web/app/{sitemap.ts => sitemap.xml/route.ts} (59%) diff --git a/apps/web/app/sitemap.xml/route.test.ts b/apps/web/app/sitemap.xml/route.test.ts new file mode 100644 index 00000000..b67e150f --- /dev/null +++ b/apps/web/app/sitemap.xml/route.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it, vi } from "vitest"; +import { GET } from "./route"; + +vi.mock("@/lib/supabase/blog-posts", () => ({ + listPublishedBlogPosts: vi.fn().mockResolvedValue([]), +})); + +describe("GET /sitemap.xml", () => { + it("returns valid sitemap XML with a Last-Modified header matching the newest entry", async () => { + const response = await GET(); + + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toBe("application/xml; charset=utf-8"); + expect(response.headers.get("cache-control")).toBe("public, max-age=0, must-revalidate"); + + const lastModifiedHeader = response.headers.get("last-modified"); + expect(lastModifiedHeader).toBeTruthy(); + expect(Number.isNaN(Date.parse(lastModifiedHeader!))).toBe(false); + + const xml = await response.text(); + expect(xml).toContain(''); + expect(xml).toContain(''); + expect(xml).toContain("https://webhooks.cc"); + expect(xml).toContain("https://webhooks.cc/webhooks/paddle"); + + const lastmodMatches = [...xml.matchAll(/([^<]+)<\/lastmod>/g)].map((m) => + new Date(m[1]).getTime() + ); + expect(lastmodMatches.length).toBeGreaterThan(0); + + const maxLastmodDate = new Date(Math.max(...lastmodMatches)); + expect(lastModifiedHeader).toBe(maxLastmodDate.toUTCString()); + }); +}); diff --git a/apps/web/app/sitemap.ts b/apps/web/app/sitemap.xml/route.ts similarity index 59% rename from apps/web/app/sitemap.ts rename to apps/web/app/sitemap.xml/route.ts index 8243901c..f25445d1 100644 --- a/apps/web/app/sitemap.ts +++ b/apps/web/app/sitemap.xml/route.ts @@ -1,4 +1,3 @@ -import type { MetadataRoute } from "next"; import { getAllDocSlugs, getDocFrontmatter } from "@/lib/docs"; import { CHANGELOG } from "@/lib/changelog"; import { LAST_CONTENT_UPDATE, PUBLIC_SITEMAP_PAGES, SITE_URL } from "@/lib/seo"; @@ -10,28 +9,33 @@ import { export const revalidate = 3600; -// One flat /sitemap.xml. It replaced a sitemap index with three child -// sitemaps: Google Search Console kept reporting the index as "Couldn't -// fetch" and never requested the children, while the site is small enough -// (well under the 50,000 URL limit) that an index buys nothing. +interface SitemapEntry { + url: string; + lastModified: Date; + changeFrequency: "always" | "hourly" | "daily" | "weekly" | "monthly" | "yearly" | "never"; + priority: number; +} + +// One flat /sitemap.xml served by an explicit Route Handler so we control the +// exact HTTP response headers (including Last-Modified and Content-Type). // // lastmod must reflect real content changes: Google stops trusting it on a site // whose dates are visibly wrong. Provider pages carry their own dates, the // /webhooks hub changes whenever one of them does, and /changelog with its // newest entry. -export default async function sitemap(): Promise { +export async function GET(): Promise { const derivedLastModified: Record = { "/webhooks": getWebhookProvidersLastModified(), "/changelog": newestChangelogDate(), }; - const pages: MetadataRoute.Sitemap = PUBLIC_SITEMAP_PAGES.map((page) => ({ + const pages: SitemapEntry[] = PUBLIC_SITEMAP_PAGES.map((page) => ({ url: page.path === "/" ? SITE_URL : `${SITE_URL}${page.path}`, lastModified: page.lastModified ?? derivedLastModified[page.path] ?? LAST_CONTENT_UPDATE, changeFrequency: page.changeFrequency, priority: page.priority, })); - const providers: MetadataRoute.Sitemap = getAllWebhookProviderPages().map((page) => ({ + const providers: SitemapEntry[] = getAllWebhookProviderPages().map((page) => ({ url: `${SITE_URL}/webhooks/${page.slug}`, lastModified: page.lastModified, changeFrequency: "monthly", @@ -40,7 +44,7 @@ export default async function sitemap(): Promise { const docSlugs = await getAllDocSlugs(); const docEntries = await Promise.all( - docSlugs.map(async (slug) => { + docSlugs.map(async (slug): Promise => { const fm = await getDocFrontmatter(slug); // A doc canonicalised to another page is not one Google should index. if (fm?.canonical) return null; @@ -52,12 +56,12 @@ export default async function sitemap(): Promise { ? fm.lastUpdated : new Date(`${fm.lastUpdated}T00:00:00.000Z`) : LAST_CONTENT_UPDATE, - changeFrequency: "monthly" as const, + changeFrequency: "monthly", priority: slug.startsWith("guides/") ? 0.8 : path === "/docs" ? 0.9 : 0.7, }; }) ); - const docs: MetadataRoute.Sitemap = docEntries.filter((entry) => entry !== null); + const docs: SitemapEntry[] = docEntries.filter((entry): entry is SitemapEntry => entry !== null); let posts: Awaited> = []; try { @@ -70,7 +74,7 @@ export default async function sitemap(): Promise { const d = new Date(post.updatedAt); return d > latest ? d : latest; }, LAST_CONTENT_UPDATE); - const blog: MetadataRoute.Sitemap = [ + const blog: SitemapEntry[] = [ { url: `${SITE_URL}/blog`, lastModified: blogLastmod, @@ -85,7 +89,45 @@ export default async function sitemap(): Promise { })), ]; - return [...pages, ...providers, ...docs, ...blog]; + const entries = [...pages, ...providers, ...docs, ...blog]; + const latestModified = entries.reduce( + (latest, entry) => (entry.lastModified > latest ? entry.lastModified : latest), + LAST_CONTENT_UPDATE + ); + + const xml = renderSitemapXml(entries); + + return new Response(xml, { + headers: { + "Content-Type": "application/xml; charset=utf-8", + "Cache-Control": "public, max-age=0, must-revalidate", + "Last-Modified": latestModified.toUTCString(), + }, + }); +} + +function renderSitemapXml(entries: readonly SitemapEntry[]): string { + const urls = entries + .map( + (entry) => ` +${escapeXml(entry.url)} +${entry.lastModified.toISOString()} +${entry.changeFrequency} +${entry.priority} +` + ) + .join("\n"); + + return `\n\n${urls}\n\n`; +} + +function escapeXml(value: string): string { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); } function newestChangelogDate(): Date { diff --git a/apps/web/lib/agent/pow.test.ts b/apps/web/lib/agent/pow.test.ts index 6b6d851c..6c828503 100644 --- a/apps/web/lib/agent/pow.test.ts +++ b/apps/web/lib/agent/pow.test.ts @@ -1,4 +1,4 @@ -import { createHmac } from "node:crypto"; +import { createHash, createHmac } from "node:crypto"; import { describe, expect, test } from "vitest"; import { adaptiveDifficulty, @@ -139,8 +139,19 @@ describe("verifySolution", () => { ok: false, reason: "bad_nonce", }); - const shifted = [nonces[1], nonces[0], nonces[2], nonces[3]]; - expect(verify(c.challenge, shifted)).toEqual({ ok: false, reason: "insufficient_work" }); + let badFirstNonce = 0; + while ( + hasLeadingZeroBits( + createHash("sha256").update(`${c.challenge}.0.${badFirstNonce}`).digest(), + c.difficulty + ) + ) { + badFirstNonce += 1; + } + expect(verify(c.challenge, [String(badFirstNonce), ...nonces.slice(1)])).toEqual({ + ok: false, + reason: "insufficient_work", + }); }); test("refuses malformed input", () => { diff --git a/apps/web/lib/changelog.ts b/apps/web/lib/changelog.ts index 4d35cbb9..6ed88555 100644 --- a/apps/web/lib/changelog.ts +++ b/apps/web/lib/changelog.ts @@ -15,13 +15,23 @@ export const TRACK_LABELS: Record = { mcp: "MCP", }; -export const APP_VERSION = "0.40.2"; +export const APP_VERSION = "0.40.3"; export const CLI_VERSION = "1.4.1"; export const SDK_VERSION = "1.15.0"; export const MCP_VERSION = "1.12.1"; export const CHANGELOG: ChangelogEntry[] = [ // ─── Web App ──────────────────────────────────────────────────────── + { + version: "0.40.3", + date: "2026-10-20", + title: "Sitemap Last-Modified Header and Deterministic PoW Unit Test", + track: "web", + items: [ + "Serve /sitemap.xml from an explicit route handler with a Last-Modified HTTP response header set to the newest entry timestamp", + "Make the proof-of-work insufficient-work unit test deterministic across random challenges", + ], + }, { version: "0.40.2", date: "2026-10-20", diff --git a/apps/web/lib/seo.ts b/apps/web/lib/seo.ts index fce25f1a..191b8b57 100644 --- a/apps/web/lib/seo.ts +++ b/apps/web/lib/seo.ts @@ -31,7 +31,7 @@ export interface SitemapPageDefinition { export const PUBLIC_SITEMAP_PAGES: readonly SitemapPageDefinition[] = [ { path: "/", changeFrequency: "weekly", priority: 1, lastModified: SEO_PASS_UPDATED }, // /go permanently redirects to / (the landing page is the guest dashboard now) - // Individual /webhooks/[provider] pages are appended in app/sitemap.ts + // Individual /webhooks/[provider] pages are appended in app/sitemap.xml/route.ts { path: "/webhooks", changeFrequency: "monthly", priority: 0.8 }, { path: "/email-testing", @@ -46,7 +46,7 @@ export const PUBLIC_SITEMAP_PAGES: readonly SitemapPageDefinition[] = [ lastModified: EMAIL_PAGES_UPDATED, }, { path: "/for-teams", changeFrequency: "monthly", priority: 0.8, lastModified: SEO_PASS_UPDATED }, - // Docs pages are auto-generated from content/docs/ in app/sitemap.ts + // Docs pages are auto-generated from content/docs/ in app/sitemap.xml/route.ts { path: "/compare", changeFrequency: "monthly", priority: 0.7, lastModified: SEO_PASS_UPDATED }, { path: "/compare/webhook-site", @@ -97,7 +97,7 @@ export const PUBLIC_SITEMAP_PAGES: readonly SitemapPageDefinition[] = [ priority: 0.6, lastModified: SEO_PASS_UPDATED, }, - // Blog pages are dynamically generated from Supabase in app/sitemap.ts + // Blog pages are dynamically generated from Supabase in app/sitemap.xml/route.ts // /api-explorer is noindexed - client-only Scalar viewer, /docs/api owns this query { path: "/installation", changeFrequency: "monthly", priority: 0.8 }, // /login is noindexed - excluded from sitemap intentionally diff --git a/apps/web/package.json b/apps/web/package.json index 639da18f..72a69530 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "web", - "version": "0.40.2", + "version": "0.40.3", "private": true, "scripts": { "dev": "next dev", diff --git a/apps/web/scripts/seo-smoke.mjs b/apps/web/scripts/seo-smoke.mjs index c070cf19..baf26e9c 100644 --- a/apps/web/scripts/seo-smoke.mjs +++ b/apps/web/scripts/seo-smoke.mjs @@ -40,6 +40,12 @@ async function main() { if (!sitemap.response.ok) { throw new Error(`Failed to fetch sitemap.xml from ${baseUrl}: ${sitemap.response.status}`); } + const sitemapLastModified = sitemap.response.headers.get("last-modified"); + if (!sitemapLastModified || Number.isNaN(Date.parse(sitemapLastModified))) { + errors.push( + `/sitemap.xml: missing or invalid Last-Modified header (${sitemapLastModified || "missing"})` + ); + } const sitemapUrls = [...sitemap.text.matchAll(/([^<]+)<\/loc>/g)].map((m) => m[1]); if (sitemapUrls.length === 0) {