improvement(seo): list the v2 API as a catalog item and answer markdown 404s (#9088)

* improvement(seo): list the v2 API as a catalog item and answer markdown 404s

* fix(seo): keep the HTML 404 for clients that refuse Markdown and answer every method

Author: waleedlatif1

apps/sim/app/.well-known/api-catalog/route.ts

@@ -8,22 +8,25 @@ const API_CATALOG_CONTENT_TYPE =
   'application/linkset+json; profile="https://www.rfc-editor.org/info/rfc9727"'
 
 /**
- * RFC 9727 API catalog: one machine-readable entry point to the v2 REST API, linking its OpenAPI
- * document (`service-desc`), reference docs (`service-doc`), and health endpoint (`status`) with
- * the RFC 8631 relations. The anchor is this deployment's own origin, so a self-hosted instance
+ * RFC 9727 API catalog: one machine-readable entry point to the v2 REST API. The first context
+ * lists the API as a catalog `item` (RFC 9727 §3.1); the second links its OpenAPI document
+ * (`service-desc`), reference docs (`service-doc`), and health endpoint (`status`) with the
+ * RFC 8631 relations. Both anchors are this deployment's own origin, so a self-hosted instance
  * describes itself. The OpenAPI document is the one published contract; its `servers` names Sim
  * Cloud, so a client of another deployment uses the anchor as its base URL, as the API docs say.
  * `Link: rel="api-catalog"` makes a HEAD request discoverable too (RFC 9727 §2).
  */
 export const GET = withRouteHandler(() => {
   const baseUrl = getBaseUrl()
   const catalogUrl = `${baseUrl}/.well-known/api-catalog`
+  const apiUrl = `${baseUrl}/api/v2`
 
   return NextResponse.json(
     {
       linkset: [
+        { anchor: catalogUrl, item: [{ href: apiUrl }] },
         {
-          anchor: `${baseUrl}/api/v2`,
+          anchor: apiUrl,
           'service-desc': [
             { href: `${DOCS_URL}/openapi.json`, type: 'application/vnd.oai.openapi+json' },
           ],

apps/sim/app/markdown-not-found/route.ts

@@ -0,0 +1,37 @@
+import { toSiteUrl } from '@/lib/core/utils/urls'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { DOCS_URL } from '@/lib/help-links'
+
+const NOT_FOUND_MARKDOWN = `# Page not found
+
+There is no page at this address. It may have moved or never existed.
+
+- [Sim home](${toSiteUrl('/')})
+- [Documentation](${DOCS_URL})
+- [llms.txt](${toSiteUrl('/llms.txt')}): an index of every public page
+- [Sitemap](${toSiteUrl('/sitemap.xml')})
+`
+
+/**
+ * The 404 body for a client that asks for Markdown. `next.config.ts` rewrites an unmatched path
+ * here, whatever its method, only when its `Accept` header names `text/markdown`, so browsers keep
+ * the HTML `not-found` page. Uncached like that page, and `Vary: Accept` because the same URL
+ * answers both ways.
+ */
+const notFound = withRouteHandler(
+  () =>
+    new Response(NOT_FOUND_MARKDOWN, {
+      status: 404,
+      headers: {
+        'Content-Type': 'text/markdown; charset=utf-8',
+        'Cache-Control': 'no-store',
+        Vary: 'Accept',
+      },
+    })
+)
+
+export const GET = notFound
+export const POST = notFound
+export const PUT = notFound
+export const PATCH = notFound
+export const DELETE = notFound

apps/sim/next.config.ts

@@ -668,16 +668,40 @@ const nextConfig: NextConfig = {
     return redirects
   },
   async rewrites() {
-    return [
-      {
-        source: '/favicon.ico',
-        destination: '/icon.svg',
-      },
-      {
-        source: '/r/:shortCode',
-        destination: 'https://go.trybeluga.ai/:shortCode',
-      },
-    ]
+    return {
+      beforeFiles: [],
+      afterFiles: [
+        {
+          source: '/favicon.ico',
+          destination: '/icon.svg',
+        },
+        {
+          source: '/r/:shortCode',
+          destination: 'https://go.trybeluga.ai/:shortCode',
+        },
+      ],
+      /**
+       * Fallback rewrites run only after every page and dynamic route has failed to match, so
+       * this answers a Markdown client's 404 in Markdown without shadowing any real route.
+       * `text/markdown` must be one of the listed media ranges, and `q=0` (RFC 9110 §12.4.2:
+       * not acceptable) keeps the HTML page. Next anchors both patterns to the whole header.
+       */
+      fallback: [
+        {
+          source: '/:path*',
+          has: [{ type: 'header', key: 'accept', value: '(.*,)?\\s*text/markdown\\s*([;,].*)?' }],
+          missing: [
+            {
+              type: 'header',
+              key: 'accept',
+              value:
+                '(.*,)?\\s*text/markdown(\\s*;[^,]*?)?\\s*;\\s*q\\s*=\\s*0(\\.0{0,3})?\\s*(,.*)?',
+            },
+          ],
+          destination: '/markdown-not-found',
+        },
+      ],
+    }
   },
 }
 

apps/sim/scripts/test-mcp-host-e2e.ts

@@ -6,8 +6,8 @@ import { getErrorMessage } from '@sim/utils/errors'
 import { SIM_DOCS_URL } from '@sim/utils/site'
 
 /**
- * Exercises the public ownership challenge, host isolation, the RFC 9727 API catalog, and the
- * app-wide JSON 404 through a running local app.
+ * Exercises the public ownership challenge, host isolation, the RFC 9727 API catalog, the
+ * app-wide JSON 404, and the Markdown 404 for `Accept: text/markdown` through a running local app.
  */
 const logger = createLogger('McpHostE2E')
 const CHALLENGE_PATH = '/.well-known/openai-apps-challenge'
@@ -17,6 +17,8 @@ const PROTECTED_RESOURCE_PATH = '/.well-known/oauth-protected-resource'
 const API_CATALOG_PATH = '/.well-known/api-catalog'
 const API_CATALOG_CONTENT_TYPE =
   'application/linkset+json; profile="https://www.rfc-editor.org/info/rfc9727"'
+const MARKDOWN_ACCEPT = 'text/markdown'
+const BROWSER_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
 const startedAt = new Date().toISOString()
 
 const configuredBaseUrl = process.env.MCP_HOST_E2E_BASE_URL
@@ -43,18 +45,25 @@ const requests: {
   path: string
   userAgent: string
   host: string
+  accept: string
   status: number
 }[] = []
 
-async function request(path: string, userAgent: string, host = MCP_HOST, method = 'GET') {
+async function request(
+  path: string,
+  userAgent: string,
+  host = MCP_HOST,
+  method = 'GET',
+  accept = '*/*'
+) {
   // boundary-raw-fetch: exercise the real proxy and route over local HTTP.
   const response = await fetch(new URL(path, baseUrl), {
     method,
-    headers: { Host: host, 'User-Agent': userAgent },
+    headers: { Host: host, 'User-Agent': userAgent, Accept: accept },
     redirect: 'error',
     signal: AbortSignal.timeout(120_000),
   })
-  requests.push({ method, path, userAgent, host, status: response.status })
+  requests.push({ method, path, userAgent, host, accept, status: response.status })
   const body = await response.text()
   return { response, body }
 }
@@ -122,7 +131,11 @@ try {
         response.headers.get('link'),
         `<${baseUrl.origin}${API_CATALOG_PATH}>; rel="api-catalog"`
       )
-      const [api] = JSON.parse(body).linkset
+      const [catalog, api] = JSON.parse(body).linkset
+      assert.deepEqual(catalog, {
+        anchor: `${baseUrl.origin}${API_CATALOG_PATH}`,
+        item: [{ href: `${baseUrl.origin}/api/v2` }],
+      })
       assert.equal(api.anchor, `${baseUrl.origin}/api/v2`)
       assert.equal(api['service-desc'][0].href, `${SIM_DOCS_URL}/openapi.json`)
       assert.equal(api.status[0].href, `${baseUrl.origin}/api/health`)
@@ -138,6 +151,77 @@ try {
     assert.match(response.headers.get('link') ?? '', /rel="api-catalog"$/)
   })
 
+  for (const path of ['/not-a-real-page', '/not-a-real/nested/page']) {
+    await check(`unknown page ${path} answers a Markdown client in Markdown`, async () => {
+      const { response, body } = await request(
+        path,
+        'python-requests/2.32.3',
+        baseUrl.host,
+        'GET',
+        MARKDOWN_ACCEPT
+      )
+      assert.equal(response.status, 404)
+      assert.equal(response.headers.get('content-type'), 'text/markdown; charset=utf-8')
+      assert.match(response.headers.get('vary') ?? '', /\bAccept\b/)
+      assert.match(response.headers.get('cache-control') ?? '', /\bno-store\b/)
+      assert.match(body, /^# Page not found\n/)
+      assert.match(body, /\/llms\.txt\)/)
+    })
+  }
+
+  await check('an unknown page answers a Markdown POST in Markdown, not 405', async () => {
+    const { response } = await request(
+      '/not-a-real-page',
+      'python-requests/2.32.3',
+      baseUrl.host,
+      'POST',
+      MARKDOWN_ACCEPT
+    )
+    assert.equal(response.status, 404)
+    assert.equal(response.headers.get('content-type'), 'text/markdown; charset=utf-8')
+  })
+
+  await check('a client that refuses Markdown with q=0 gets the HTML 404', async () => {
+    const { response } = await request(
+      '/not-a-real-page',
+      'python-requests/2.32.3',
+      baseUrl.host,
+      'GET',
+      'text/html, text/markdown;q=0'
+    )
+    assert.equal(response.status, 404)
+    assert.match(response.headers.get('content-type') ?? '', /^text\/html/)
+  })
+
+  await check('unknown page still answers a browser with the HTML 404', async () => {
+    const { response } = await request(
+      '/not-a-real-page',
+      'Mozilla/5.0',
+      baseUrl.host,
+      'GET',
+      BROWSER_ACCEPT
+    )
+    assert.equal(response.status, 404)
+    assert.match(response.headers.get('content-type') ?? '', /^text\/html/)
+  })
+
+  await check('a real page is not shadowed by the Markdown 404', async () => {
+    const { response } = await request('/', 'Mozilla/5.0', baseUrl.host, 'GET', MARKDOWN_ACCEPT)
+    assert.equal(response.status, 200)
+  })
+
+  await check('an unknown API path keeps its JSON 404 for a Markdown client', async () => {
+    const { response, body } = await request(
+      '/api/not-a-real-route',
+      'Mozilla/5.0',
+      baseUrl.host,
+      'GET',
+      MARKDOWN_ACCEPT
+    )
+    assert.equal(response.status, 404)
+    assert.deepEqual(JSON.parse(body), { error: 'Not found' })
+  })
+
   await check('application host still serves health checks', async () => {
     const { response } = await request('/api/health', 'Mozilla/5.0', baseUrl.host)
     assert.equal(response.status, 200)