fix(web): serve /sitemap.xml with Last-Modified header (web 0.40.3)

apps/web/app/sitemap.xml/route.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it, vi } from "vitest";
+import { GET } from "./route";
+
+vi.mock("@/lib/supabase/blog-posts", () => ({
+  listPublishedBlogPosts: vi.fn().mockResolvedValue([]),
+}));
+
+describe("GET /sitemap.xml", () => {
+  it("returns valid sitemap XML with a Last-Modified header matching the newest entry", async () => {
+    const response = await GET();
+
+    expect(response.status).toBe(200);
+    expect(response.headers.get("content-type")).toBe("application/xml; charset=utf-8");
+    expect(response.headers.get("cache-control")).toBe("public, max-age=0, must-revalidate");
+
+    const lastModifiedHeader = response.headers.get("last-modified");
+    expect(lastModifiedHeader).toBeTruthy();
+    expect(Number.isNaN(Date.parse(lastModifiedHeader!))).toBe(false);
+
+    const xml = await response.text();
+    expect(xml).toContain('<?xml version="1.0" encoding="UTF-8"?>');
+    expect(xml).toContain('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">');
+    expect(xml).toContain("<loc>https://webhooks.cc</loc>");
+    expect(xml).toContain("<loc>https://webhooks.cc/webhooks/paddle</loc>");
+
+    const lastmodMatches = [...xml.matchAll(/<lastmod>([^<]+)<\/lastmod>/g)].map((m) =>
+      new Date(m[1]).getTime()
+    );
+    expect(lastmodMatches.length).toBeGreaterThan(0);
+
+    const maxLastmodDate = new Date(Math.max(...lastmodMatches));
+    expect(lastModifiedHeader).toBe(maxLastmodDate.toUTCString());
+  });
+});

apps/web/app/sitemap.ts → apps/web/app/sitemap.xml/route.ts

@@ -1,4 +1,3 @@
-import type { MetadataRoute } from "next";
 import { getAllDocSlugs, getDocFrontmatter } from "@/lib/docs";
 import { CHANGELOG } from "@/lib/changelog";
 import { LAST_CONTENT_UPDATE, PUBLIC_SITEMAP_PAGES, SITE_URL } from "@/lib/seo";
@@ -10,28 +9,33 @@ import {
 
 export const revalidate = 3600;
 
-// One flat /sitemap.xml. It replaced a sitemap index with three child
-// sitemaps: Google Search Console kept reporting the index as "Couldn't
-// fetch" and never requested the children, while the site is small enough
-// (well under the 50,000 URL limit) that an index buys nothing.
+interface SitemapEntry {
+  url: string;
+  lastModified: Date;
+  changeFrequency: "always" | "hourly" | "daily" | "weekly" | "monthly" | "yearly" | "never";
+  priority: number;
+}
+
+// One flat /sitemap.xml served by an explicit Route Handler so we control the
+// exact HTTP response headers (including Last-Modified and Content-Type).
 //
 // lastmod must reflect real content changes: Google stops trusting it on a site
 // whose dates are visibly wrong. Provider pages carry their own dates, the
 // /webhooks hub changes whenever one of them does, and /changelog with its
 // newest entry.
-export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
+export async function GET(): Promise<Response> {
   const derivedLastModified: Record<string, Date> = {
     "/webhooks": getWebhookProvidersLastModified(),
     "/changelog": newestChangelogDate(),
   };
-  const pages: MetadataRoute.Sitemap = PUBLIC_SITEMAP_PAGES.map((page) => ({
+  const pages: SitemapEntry[] = PUBLIC_SITEMAP_PAGES.map((page) => ({
     url: page.path === "/" ? SITE_URL : `${SITE_URL}${page.path}`,
     lastModified: page.lastModified ?? derivedLastModified[page.path] ?? LAST_CONTENT_UPDATE,
     changeFrequency: page.changeFrequency,
     priority: page.priority,
   }));
 
-  const providers: MetadataRoute.Sitemap = getAllWebhookProviderPages().map((page) => ({
+  const providers: SitemapEntry[] = getAllWebhookProviderPages().map((page) => ({
     url: `${SITE_URL}/webhooks/${page.slug}`,
     lastModified: page.lastModified,
     changeFrequency: "monthly",
@@ -40,7 +44,7 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
 
   const docSlugs = await getAllDocSlugs();
   const docEntries = await Promise.all(
-    docSlugs.map(async (slug) => {
+    docSlugs.map(async (slug): Promise<SitemapEntry | null> => {
       const fm = await getDocFrontmatter(slug);
       // A doc canonicalised to another page is not one Google should index.
       if (fm?.canonical) return null;
@@ -52,12 +56,12 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
             ? fm.lastUpdated
             : new Date(`${fm.lastUpdated}T00:00:00.000Z`)
           : LAST_CONTENT_UPDATE,
-        changeFrequency: "monthly" as const,
+        changeFrequency: "monthly",
         priority: slug.startsWith("guides/") ? 0.8 : path === "/docs" ? 0.9 : 0.7,
       };
     })
   );
-  const docs: MetadataRoute.Sitemap = docEntries.filter((entry) => entry !== null);
+  const docs: SitemapEntry[] = docEntries.filter((entry): entry is SitemapEntry => entry !== null);
 
   let posts: Awaited<ReturnType<typeof listPublishedBlogPosts>> = [];
   try {
@@ -70,7 +74,7 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
     const d = new Date(post.updatedAt);
     return d > latest ? d : latest;
   }, LAST_CONTENT_UPDATE);
-  const blog: MetadataRoute.Sitemap = [
+  const blog: SitemapEntry[] = [
     {
       url: `${SITE_URL}/blog`,
       lastModified: blogLastmod,
@@ -85,7 +89,45 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
     })),
   ];
 
-  return [...pages, ...providers, ...docs, ...blog];
+  const entries = [...pages, ...providers, ...docs, ...blog];
+  const latestModified = entries.reduce<Date>(
+    (latest, entry) => (entry.lastModified > latest ? entry.lastModified : latest),
+    LAST_CONTENT_UPDATE
+  );
+
+  const xml = renderSitemapXml(entries);
+
+  return new Response(xml, {
+    headers: {
+      "Content-Type": "application/xml; charset=utf-8",
+      "Cache-Control": "public, max-age=0, must-revalidate",
+      "Last-Modified": latestModified.toUTCString(),
+    },
+  });
+}
+
+function renderSitemapXml(entries: readonly SitemapEntry[]): string {
+  const urls = entries
+    .map(
+      (entry) => `<url>
+<loc>${escapeXml(entry.url)}</loc>
+<lastmod>${entry.lastModified.toISOString()}</lastmod>
+<changefreq>${entry.changeFrequency}</changefreq>
+<priority>${entry.priority}</priority>
+</url>`
+    )
+    .join("\n");
+
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls}\n</urlset>\n`;
+}
+
+function escapeXml(value: string): string {
+  return value
+    .replaceAll("&", "&amp;")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+    .replaceAll('"', "&quot;")
+    .replaceAll("'", "&apos;");
 }
 
 function newestChangelogDate(): Date {

apps/web/lib/agent/pow.test.ts

@@ -1,4 +1,4 @@
-import { createHmac } from "node:crypto";
+import { createHash, createHmac } from "node:crypto";
 import { describe, expect, test } from "vitest";
 import {
   adaptiveDifficulty,
@@ -139,8 +139,19 @@ describe("verifySolution", () => {
       ok: false,
       reason: "bad_nonce",
     });
-    const shifted = [nonces[1], nonces[0], nonces[2], nonces[3]];
-    expect(verify(c.challenge, shifted)).toEqual({ ok: false, reason: "insufficient_work" });
+    let badFirstNonce = 0;
+    while (
+      hasLeadingZeroBits(
+        createHash("sha256").update(`${c.challenge}.0.${badFirstNonce}`).digest(),
+        c.difficulty
+      )
+    ) {
+      badFirstNonce += 1;
+    }
+    expect(verify(c.challenge, [String(badFirstNonce), ...nonces.slice(1)])).toEqual({
+      ok: false,
+      reason: "insufficient_work",
+    });
   });
 
   test("refuses malformed input", () => {

apps/web/lib/changelog.ts

@@ -15,13 +15,23 @@ export const TRACK_LABELS: Record<ChangelogTrack, string> = {
   mcp: "MCP",
 };
 
-export const APP_VERSION = "0.40.2";
+export const APP_VERSION = "0.40.3";
 export const CLI_VERSION = "1.4.1";
 export const SDK_VERSION = "1.15.0";
 export const MCP_VERSION = "1.12.1";
 
 export const CHANGELOG: ChangelogEntry[] = [
   // ─── Web App ────────────────────────────────────────────────────────
+  {
+    version: "0.40.3",
+    date: "2026-10-20",
+    title: "Sitemap Last-Modified Header and Deterministic PoW Unit Test",
+    track: "web",
+    items: [
+      "Serve /sitemap.xml from an explicit route handler with a Last-Modified HTTP response header set to the newest entry timestamp",
+      "Make the proof-of-work insufficient-work unit test deterministic across random challenges",
+    ],
+  },
   {
     version: "0.40.2",
     date: "2026-10-20",

apps/web/lib/seo.ts

@@ -31,7 +31,7 @@ export interface SitemapPageDefinition {
 export const PUBLIC_SITEMAP_PAGES: readonly SitemapPageDefinition[] = [
   { path: "/", changeFrequency: "weekly", priority: 1, lastModified: SEO_PASS_UPDATED },
   // /go permanently redirects to / (the landing page is the guest dashboard now)
-  // Individual /webhooks/[provider] pages are appended in app/sitemap.ts
+  // Individual /webhooks/[provider] pages are appended in app/sitemap.xml/route.ts
   { path: "/webhooks", changeFrequency: "monthly", priority: 0.8 },
   {
     path: "/email-testing",
@@ -46,7 +46,7 @@ export const PUBLIC_SITEMAP_PAGES: readonly SitemapPageDefinition[] = [
     lastModified: EMAIL_PAGES_UPDATED,
   },
   { path: "/for-teams", changeFrequency: "monthly", priority: 0.8, lastModified: SEO_PASS_UPDATED },
-  // Docs pages are auto-generated from content/docs/ in app/sitemap.ts
+  // Docs pages are auto-generated from content/docs/ in app/sitemap.xml/route.ts
   { path: "/compare", changeFrequency: "monthly", priority: 0.7, lastModified: SEO_PASS_UPDATED },
   {
     path: "/compare/webhook-site",
@@ -97,7 +97,7 @@ export const PUBLIC_SITEMAP_PAGES: readonly SitemapPageDefinition[] = [
     priority: 0.6,
     lastModified: SEO_PASS_UPDATED,
   },
-  // Blog pages are dynamically generated from Supabase in app/sitemap.ts
+  // Blog pages are dynamically generated from Supabase in app/sitemap.xml/route.ts
   // /api-explorer is noindexed - client-only Scalar viewer, /docs/api owns this query
   { path: "/installation", changeFrequency: "monthly", priority: 0.8 },
   // /login is noindexed - excluded from sitemap intentionally

apps/web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "0.40.2",
+  "version": "0.40.3",
   "private": true,
   "scripts": {
     "dev": "next dev",

apps/web/scripts/seo-smoke.mjs

@@ -40,6 +40,12 @@ async function main() {
   if (!sitemap.response.ok) {
     throw new Error(`Failed to fetch sitemap.xml from ${baseUrl}: ${sitemap.response.status}`);
   }
+  const sitemapLastModified = sitemap.response.headers.get("last-modified");
+  if (!sitemapLastModified || Number.isNaN(Date.parse(sitemapLastModified))) {
+    errors.push(
+      `/sitemap.xml: missing or invalid Last-Modified header (${sitemapLastModified || "missing"})`
+    );
+  }
   const sitemapUrls = [...sitemap.text.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]);
 
   if (sitemapUrls.length === 0) {