improvement(library): single JSON-LD source, Markdown articles, and IndexNow pings

Author: waleedlatif1

.github/workflows/ci.yml

@@ -636,6 +636,42 @@ jobs:
           fi
           bunx trigger.dev@4.5.16 promote "$VERSION" "${TARGET_ARGS[@]}"
 
+  # Production only, once the new app serves traffic: submit the library and comparison URLs this
+  # push changed to IndexNow. Best effort: nothing depends on it, and a failure never fails the run.
+  indexnow:
+    name: indexnow
+    needs: [promote, trigger-promote]
+    if: >-
+      !cancelled() &&
+      github.event_name == 'push' &&
+      github.ref == 'refs/heads/main' &&
+      needs.promote.outputs.promoted == 'true' &&
+      needs.trigger-promote.result == 'success'
+    runs-on: *runner-2vcpu
+    timeout-minutes: 5
+    continue-on-error: true
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: 1.4.4
+
+      # The diff needs only the two trees, so the previous head is fetched alone. A push that
+      # created the branch has an all-zero `before`, which the script skips.
+      - name: Submit changed URLs to IndexNow
+        env:
+          BASE_SHA: ${{ github.event.before }}
+        run: |
+          if [[ ! "$BASE_SHA" =~ ^0+$ ]]; then
+            git fetch --no-tags --depth=1 origin "$BASE_SHA"
+          fi
+          bun scripts/ping-indexnow.ts "$BASE_SHA" "$GITHUB_SHA"
+
   # Build ARM64 images for GHCR (main branch only, runs in parallel with
   # tests). Pushes only the immutable sha tag — latest-arm64/version-arm64
   # are applied by ghcr after the gate, so a failing run

apps/sim/app/(landing)/blog/[slug]/page.tsx

@@ -24,7 +24,7 @@ export async function generateMetadata({
   const { slug } = await params
   const post = await getPostBySlug(slug)
   if (!post || post.draft) return {}
-  return buildPostMetadata(post)
+  return buildPostMetadata(post, BLOG_SECTION)
 }
 
 export const revalidate = 86400

apps/sim/app/(landing)/changelog/[slug]/page.tsx

@@ -19,7 +19,7 @@ export async function generateStaticParams() {
 export async function generateMetadata({ params }: ChangelogEntryPageProps): Promise<Metadata> {
   const { slug } = await params
   const entry = await getEntryBySlug(slug)
-  return entry ? buildPostMetadata(entry) : {}
+  return entry ? buildPostMetadata(entry, CHANGELOG_SECTION) : {}
 }
 
 export default async function ChangelogEntryPage({ params }: ChangelogEntryPageProps) {

apps/sim/app/(landing)/components/content-post-page/content-post-page.tsx

@@ -40,7 +40,7 @@ export function ContentPostPage({
   const showUpdated = modifiedIso.slice(0, 10) !== post.date.slice(0, 10)
 
   return (
-    <article className='w-full bg-[var(--bg)]' itemScope itemType='https://schema.org/BlogPosting'>
+    <article className='w-full bg-[var(--bg)]'>
       <JsonLd data={graphJsonLd} />
       <header className='mx-auto w-full max-w-[1728px] px-10 pt-[112px] max-sm:pt-20 max-md:px-7 max-lg:px-8 max-xl:px-9'>
         <div className='mb-6'>
@@ -59,7 +59,6 @@ export function ContentPostPage({
                 sizes='(max-width: 1023px) calc(100vw - 64px), 450px'
                 priority
                 fetchPriority='high'
-                itemProp='image'
                 unoptimized
               />
             </div>
@@ -68,13 +67,13 @@ export function ContentPostPage({
             <div>
               <h1
                 className='text-balance text-[28px] text-[var(--text-primary)] leading-[110%] tracking-[-0.02em] sm:text-[36px] md:text-[44px] lg:text-[52px]'
-                itemProp='headline'
+                data-speakable='headline'
               >
                 {post.title}
               </h1>
               <p
                 className='mt-4 text-[var(--text-body)] text-base leading-[150%] tracking-[0.02em] sm:text-lg sm:leading-7'
-                itemProp='description'
+                data-speakable='description'
               >
                 {post.description}
               </p>
@@ -84,25 +83,21 @@ export function ContentPostPage({
                 <time
                   className='text-[var(--text-secondary)] text-xs uppercase tracking-[0.1em]'
                   dateTime={post.date}
-                  itemProp='datePublished'
                 >
                   {formatPostDate(post.date)}
                 </time>
-                {showUpdated ? (
+                {showUpdated && (
                   <>
                     <span aria-hidden='true' className='text-[var(--text-secondary)] text-xs'>
                       ·
                     </span>
                     <time
                       className='text-[var(--text-secondary)] text-xs uppercase tracking-[0.1em]'
                       dateTime={modifiedIso}
-                      itemProp='dateModified'
                     >
                       Updated {formatPostDate(modifiedIso)}
                     </time>
                   </>
-                ) : (
-                  <meta itemProp='dateModified' content={modifiedIso} />
                 )}
               </div>
               <div className='flex items-center gap-3'>
@@ -118,11 +113,8 @@ export function ContentPostPage({
                       href={`${basePath}/authors/${encodeURIComponent(a?.id ?? '')}`}
                       rel='author'
                       className='text-[var(--text-secondary)] text-xs uppercase tracking-[0.1em] hover:text-[var(--text-primary)]'
-                      itemProp='author'
-                      itemScope
-                      itemType='https://schema.org/Person'
                     >
-                      <span itemProp='name'>{a?.name}</span>
+                      {a?.name}
                     </Link>
                   </div>
                 ))}
@@ -139,7 +131,7 @@ export function ContentPostPage({
 
       <div className='mx-auto w-full max-w-[1728px] px-10 max-md:px-7 max-lg:px-8 max-xl:px-9'>
         <div className='border-[var(--border)] border-x'>
-          <div className='mx-auto max-w-[900px] px-6 py-16' itemProp='articleBody'>
+          <div className='mx-auto max-w-[900px] px-6 py-16'>
             <div
               className={cn(
                 'prose prose-lg max-w-none prose-headings:font-season prose-headings:tracking-[-0.02em]',
@@ -169,11 +161,6 @@ export function ContentPostPage({
       </div>
 
       <div className='-mt-px h-px w-full bg-[var(--border)]' />
-
-      <meta itemProp='publisher' content='Sim' />
-      <meta itemProp='inLanguage' content='en-US' />
-      <meta itemProp='keywords' content={post.tags.join(', ')} />
-      {post.wordCount && <meta itemProp='wordCount' content={String(post.wordCount)} />}
     </article>
   )
 }

apps/sim/app/(landing)/customers/[slug]/page.tsx

@@ -21,7 +21,7 @@ export async function generateMetadata({ params }: CustomerPageProps): Promise<M
   const { slug } = await params
   if (!CUSTOMER_STORIES.some((story) => story.slug === slug)) return {}
   const post = await getCustomerStoryBySlug(slug)
-  return post ? buildPostMetadata(post) : {}
+  return post ? buildPostMetadata(post, CUSTOMER_SECTION) : {}
 }
 
 export default async function Page({ params }: CustomerPageProps) {

apps/sim/app/library.md/[slug]/route.ts

@@ -0,0 +1,57 @@
+import type { NextRequest } from 'next/server'
+import type { ContentMeta } from '@/lib/content/schema'
+import { toSiteUrl } from '@/lib/core/utils/urls'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { getAllPostMeta, getPostSource } from '@/lib/library/registry'
+import { LIBRARY_SECTION } from '@/lib/library/seo'
+import { markdownResponse, section, toLlmsMarkdown } from '@/app/llms.txt/llms'
+
+type RouteContext = { params: Promise<{ slug: string }> }
+
+const NOT_FOUND_MARKDOWN = `# Article not found
+
+There is no library article at this address. Browse the [library](${toSiteUrl(LIBRARY_SECTION.basePath)}) or the [llms.txt](${toSiteUrl('/llms.txt')}) index.
+`
+
+function postMarkdownBlocks(post: ContentMeta, source: string): string[] {
+  const authors = (post.authors.length > 0 ? post.authors : [post.author]).map((a) => a.name)
+  const published = post.date.slice(0, 10)
+  const updated = (post.updated ?? post.date).slice(0, 10)
+  return [
+    `# ${post.title}`,
+    `> ${post.description}`,
+    [
+      `URL: ${post.canonical}`,
+      `By ${authors.join(', ')} · Published ${published}${updated !== published ? ` · Updated ${updated}` : ''}`,
+    ].join('\n'),
+    toLlmsMarkdown(source, 0),
+    section(
+      'FAQ',
+      (post.faq ?? []).map((item) => `### ${item.q}\n\n${item.a}`),
+      '\n\n'
+    ),
+  ]
+}
+
+/**
+ * A library article as Markdown, at `/library/<slug>.md` (and at `/library/<slug>` for a client
+ * whose `Accept` names `text/markdown`), via the rewrites in `next.config.ts`. Drafts and unknown
+ * slugs are a Markdown 404. Uncached, like the HTML post page: the same body answers at the post's
+ * own URL, so a cache that ignores `Vary` must never keep it.
+ */
+export const GET = withRouteHandler(async (_request: NextRequest, context: RouteContext) => {
+  const { slug } = await context.params
+  const [posts, source] = await Promise.all([getAllPostMeta(), getPostSource(slug)])
+  const post = posts.find((candidate) => candidate.slug === slug)
+  const response =
+    post && source !== null
+      ? markdownResponse(postMarkdownBlocks(post, source), 0)
+      : new Response(NOT_FOUND_MARKDOWN, {
+          status: 404,
+          headers: { 'Content-Type': 'text/markdown; charset=utf-8' },
+        })
+  response.headers.set('Cache-Control', 'no-store')
+  response.headers.set('Vary', 'Accept')
+  if (post) response.headers.set('Link', `<${post.canonical}>; rel="canonical"`)
+  return response
+})

apps/sim/app/llms.txt/route.ts

@@ -79,6 +79,8 @@ export async function GET() {
         ),
       ]),
       section('Library', [
+        'Each article is also served as Markdown at its URL with `.md` appended.',
+        '',
         linkLine('Library', '/library', 'Comparisons, how-tos, and roundups on building AI agents'),
         ...libraryPosts.map((p) => linkLine(p.title, p.canonical, p.description)),
       ]),

apps/sim/lib/blog/seo.ts

@@ -17,6 +17,7 @@ export const BLOG_SECTION: ContentSection = {
   name: 'Blog',
   basePath: '/blog',
   feedPath: '/blog/rss.xml',
+  hasAuthorPages: true,
   description: 'Announcements, insights, and guides for building AI agents.',
 }
 

apps/sim/lib/content/faq.tsx

@@ -1,19 +1,13 @@
 export function FAQ({ items }: { items: { q: string; a: string }[] }) {
   if (!items || items.length === 0) return null
   return (
-    <section className='mt-12' itemScope itemType='https://schema.org/FAQPage'>
+    <section className='mt-12'>
       <h2 className='mb-4 font-medium text-[24px] text-[var(--text-primary)]'>FAQ</h2>
       <div className='space-y-6'>
         {items.map((it, i) => (
-          <div key={i} itemScope itemType='https://schema.org/Question' itemProp='mainEntity'>
-            <h3 className='mb-2 font-medium text-[20px] text-[var(--text-primary)]' itemProp='name'>
-              {it.q}
-            </h3>
-            <div itemScope itemType='https://schema.org/Answer' itemProp='acceptedAnswer'>
-              <p className='text-[19px] text-[var(--text-subtle)] leading-relaxed' itemProp='text'>
-                {it.a}
-              </p>
-            </div>
+          <div key={i}>
+            <h3 className='mb-2 font-medium text-[20px] text-[var(--text-primary)]'>{it.q}</h3>
+            <p className='text-[19px] text-[var(--text-subtle)] leading-relaxed'>{it.a}</p>
           </div>
         ))}
       </div>