fix(library): noindex direct Markdown article URLs on self-hosted deployments and diff IndexNow from the last submitted revision

Author: waleedlatif1

.github/workflows/ci.yml

@@ -656,6 +656,7 @@ jobs:
     continue-on-error: true
     permissions:
       contents: read
+      actions: read
     steps:
       - name: Checkout code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
@@ -665,12 +666,28 @@ jobs:
         with:
           bun-version: 1.4.4
 
-      # The diff needs only the two trees, so the previous head is fetched alone. A push that
-      # created the branch has an all-zero `before`, which the script skips.
+      # Diff from the head of the last earlier run whose submission succeeded, not the previous
+      # push: a push that never deployed, or whose submission failed, would otherwise drop its
+      # changed URLs for good. Falls back to the previous push when no such run is in view. The
+      # diff needs only the two trees, so that one commit is fetched alone; an all-zero `before`
+      # (the branch's first push) has no base, which the script skips.
       - name: Submit changed URLs to IndexNow
         env:
-          BASE_SHA: ${{ github.event.before }}
+          GH_TOKEN: ${{ github.token }}
+          PUSH_BEFORE: ${{ github.event.before }}
         run: |
+          BASE_SHA=""
+          while read -r run_id head_sha; do
+            conclusion="$(gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/jobs?per_page=100" \
+              --jq '.jobs[] | select(.name == "indexnow") | .conclusion')"
+            if [[ "$conclusion" == "success" ]]; then
+              BASE_SHA="$head_sha"
+              break
+            fi
+          done < <(gh api "repos/$GITHUB_REPOSITORY/actions/workflows/ci.yml/runs?branch=main&event=push&per_page=30" \
+            --jq ".workflow_runs[] | select(.id < $GITHUB_RUN_ID) | \"\\(.id) \\(.head_sha)\"")
+          BASE_SHA="${BASE_SHA:-$PUSH_BEFORE}"
+          echo "Submitting URLs changed in $BASE_SHA..$GITHUB_SHA"
           if [[ ! "$BASE_SHA" =~ ^0+$ ]]; then
             git fetch --no-tags --depth=1 origin "$BASE_SHA"
           fi

apps/sim/lib/landing/routes.ts

@@ -70,7 +70,14 @@ const MARKETING_DOCUMENT_PATHS: ReadonlySet<string> = new Set([
   '/changelog.xml',
 ])
 
+/** Top-level route segments that republish landing content as documents, e.g. `/library.md/<slug>`. */
+const MARKETING_DOCUMENT_SEGMENTS: ReadonlySet<string> = new Set(['library.md'])
+
 /** Whether a pathname serves marketing content: a landing page or a marketing document. */
 export function isMarketingPath(pathname: string): boolean {
-  return isLandingPath(pathname) || MARKETING_DOCUMENT_PATHS.has(pathname)
+  return (
+    isLandingPath(pathname) ||
+    MARKETING_DOCUMENT_PATHS.has(pathname) ||
+    MARKETING_DOCUMENT_SEGMENTS.has(pathname.split('/')[1])
+  )
 }

apps/sim/proxy.test.ts

@@ -179,6 +179,7 @@ describe('proxy indexing policy', () => {
     expect(robotsTag('/llms.txt')).toBe('noindex, nofollow')
     expect(robotsTag('/llms-full.txt')).toBe('noindex, nofollow')
     expect(robotsTag('/changelog.xml')).toBe('noindex, nofollow')
+    expect(robotsTag('/library.md/some-article')).toBe('noindex, nofollow')
   })
 
   it('leaves the marketing pages and documents of the hosted deployment indexable', () => {
@@ -189,6 +190,7 @@ describe('proxy indexing policy', () => {
     expect(robotsTag('/llms.txt')).toBeNull()
     expect(robotsTag('/llms-full.txt')).toBeNull()
     expect(robotsTag('/changelog.xml')).toBeNull()
+    expect(robotsTag('/library.md/some-article')).toBeNull()
   })
 
   it('leaves a self-hosted non-landing public page indexable', () => {
@@ -216,6 +218,7 @@ describe('proxy matcher', () => {
     expect(matches('/llms.txt')).toBe(true)
     expect(matches('/llms-full.txt')).toBe(true)
     expect(matches('/changelog.xml')).toBe(true)
+    expect(matches('/library.md/some-article')).toBe(true)
   })
 
   it('still runs for every other API route', () => {