diff --git a/apps/eng-docs/scripts/gather.test.ts b/apps/eng-docs/scripts/gather.test.ts new file mode 100644 index 00000000..9d62583c --- /dev/null +++ b/apps/eng-docs/scripts/gather.test.ts @@ -0,0 +1,727 @@ +import { describe, expect, test } from "bun:test"; +import { + buildFrontmatter, + buildIndex, + buildSidebar, + classify, + destRelPath, + editUrlFor, + extractTitle, + isExcluded, + parseExclusions, + rewriteLinks, + routeSlug, + stripFirstH1, + transform, +} from "./gather.ts"; + +// -- extractTitle ------------------------------------------------------------- +// The docs/ sources carry no frontmatter — every file opens with a `# H1`. +// Starlight's docsSchema requires a `title`, so the gather derives it from +// that first heading. + +describe("extractTitle", () => { + test("returns the text of the first H1", () => { + expect(extractTitle("# Technology stack (engineering)\n\nbody")).toBe( + "Technology stack (engineering)", + ); + }); + + test("ignores leading blank lines and whitespace before the H1", () => { + expect(extractTitle("\n\n # Observability\n")).toBe("Observability"); + }); + + test("takes the first H1 when several headings follow", () => { + expect(extractTitle("# First\n\n## Second\n\n# Third")).toBe("First"); + }); + + test("does not treat an H2 as the title", () => { + expect(extractTitle("## Not a title\n\n# Real title")).toBe("Real title"); + }); + + test("trims trailing whitespace off the heading text", () => { + expect(extractTitle("# Padded \n")).toBe("Padded"); + }); + + test("falls back to the basename (no extension) when there is no H1", () => { + expect( + extractTitle("no heading here\n", "docs/specs/tools/README.md"), + ).toBe("README"); + }); + + test("does not mistake a `#` inside a fenced code block for the title", () => { + const src = "```sh\n# a shell comment\n```\n\n# Real heading\n"; + expect(extractTitle(src)).toBe("Real heading"); + }); + + test("does not mistake a `#` inside a tilde-fenced code block for the title", () => { + const src = "~~~sh\n# a shell comment\n~~~\n\n# Real heading\n"; + expect(extractTitle(src)).toBe("Real heading"); + }); +}); + +// -- buildFrontmatter --------------------------------------------------------- +// Real titles contain YAML-significant characters (`:`, backticks, `&`, `(`, +// em-dash). The injected block must round-trip through a YAML parser, so the +// title is always double-quoted with embedded quotes/backslashes escaped. The +// optional second arg adds an `editUrl:` line for gathered-outside-docs pages. + +describe("buildFrontmatter", () => { + test("wraps a plain title in a double-quoted YAML scalar", () => { + expect(buildFrontmatter("Observability")).toBe( + '---\ntitle: "Observability"\n---\n', + ); + }); + + test("keeps a title containing a colon safe (would break bare YAML)", () => { + expect(buildFrontmatter("CI/CD evolution: TS dynamic pipelines")).toBe( + '---\ntitle: "CI/CD evolution: TS dynamic pipelines"\n---\n', + ); + }); + + test("escapes embedded double quotes", () => { + expect(buildFrontmatter('The "read layer"')).toBe( + '---\ntitle: "The \\"read layer\\""\n---\n', + ); + }); + + test("escapes backslashes before quotes so the escape is unambiguous", () => { + expect(buildFrontmatter("a\\b")).toBe('---\ntitle: "a\\\\b"\n---\n'); + }); + + test("leaves backticks, ampersands and parens intact inside the quotes", () => { + expect(buildFrontmatter("Per-PR candidate CI image (`:pr-`)")).toBe( + '---\ntitle: "Per-PR candidate CI image (`:pr-`)"\n---\n', + ); + }); + + test("adds an editUrl line under the title when one is given", () => { + expect( + buildFrontmatter( + "Observability", + "https://github.com/sealedsecurity/compass/edit/main/ci/README.md", + ), + ).toBe( + '---\ntitle: "Observability"\neditUrl: "https://github.com/sealedsecurity/compass/edit/main/ci/README.md"\n---\n', + ); + }); + + test("still escapes the title when an editUrl is present", () => { + expect( + buildFrontmatter( + 'The "read layer"', + "https://github.com/sealedsecurity/compass/edit/main/docs/a.md", + ), + ).toBe( + '---\ntitle: "The \\"read layer\\""\neditUrl: "https://github.com/sealedsecurity/compass/edit/main/docs/a.md"\n---\n', + ); + }); +}); + +// -- stripFirstH1 ------------------------------------------------------------- +// Starlight renders the frontmatter `title` as the page's

. Leaving the +// original `# H1` in the body would double it, so the gather removes exactly +// the first H1 line (and a single blank line after it) and nothing else. + +describe("stripFirstH1", () => { + test("removes the first H1 line", () => { + expect(stripFirstH1("# Title\n\nbody\n")).toBe("body\n"); + }); + + test("removes only ONE blank line after the H1", () => { + expect(stripFirstH1("# Title\n\n\nbody\n")).toBe("\nbody\n"); + }); + + test("leaves a later H1 in place (only the first is stripped)", () => { + expect(stripFirstH1("# One\n\nmid\n\n# Two\n")).toBe("mid\n\n# Two\n"); + }); + + test("is a no-op when the body has no H1", () => { + expect(stripFirstH1("no heading\n")).toBe("no heading\n"); + }); + + test("preserves body content that follows immediately (no blank line)", () => { + expect(stripFirstH1("# Title\nbody\n")).toBe("body\n"); + }); + + test("skips a `#` inside a leading fenced code block", () => { + const src = "```sh\n# not a heading\n```\n\n# Real Title\n\nbody\n"; + expect(stripFirstH1(src)).toBe("```sh\n# not a heading\n```\n\nbody\n"); + }); +}); + +// -- destRelPath -------------------------------------------------------------- +// The docs/ subtree layout is preserved verbatim under src/content/docs/ so +// Starlight's slug + the per-domain sidebar autogenerate match the on-disk +// taxonomy. + +describe("destRelPath", () => { + test("maps a docs/ path to the same relative path under content", () => { + expect(destRelPath("docs/designs/repo/compass-eng-docs/design.md")).toBe( + "designs/repo/compass-eng-docs/design.md", + ); + }); + + test("handles a top-level doc directly under a domain", () => { + expect(destRelPath("docs/architecture/overview.md")).toBe( + "architecture/overview.md", + ); + }); +}); + +// -- classify ----------------------------------------------------------------- +// The single source of truth for where each gathered file lands. Every path is +// routed to exactly one nav section + a collision-free dest path. + +describe("classify", () => { + test("keeps a docs/ file under its section verbatim", () => { + expect(classify("docs/designs/repo/compass-eng-docs/design.md")).toEqual({ + section: "designs", + destRel: "designs/repo/compass-eng-docs/design.md", + }); + }); + + test("routes each docs domain by its directory name", () => { + expect(classify("docs/specs/tools/setup.md").section).toBe("specs"); + expect(classify("docs/architecture/overview.md").section).toBe( + "architecture", + ); + }); + + // Regression 1: root README.md and forks/README.md share a basename and must + // NOT collide under contributing/ — the slug disambiguates by source path. + test("root README.md and forks/README.md land at different contributing dests", () => { + const root = classify("README.md"); + const forks = classify("forks/README.md"); + expect(root).toEqual({ + section: "contributing", + destRel: "contributing/README.md", + }); + expect(forks).toEqual({ + section: "contributing", + destRel: "contributing/forks-README.md", + }); + expect(root.destRel).not.toBe(forks.destRel); + }); + + test("routes the other contributing files (AGENTS, CONTRIBUTING) too", () => { + expect(classify("AGENTS.md")).toEqual({ + section: "contributing", + destRel: "contributing/AGENTS.md", + }); + expect(classify("CONTRIBUTING.md")).toEqual({ + section: "contributing", + destRel: "contributing/CONTRIBUTING.md", + }); + }); + + // Regression 2: a top-level package file drops its dir exactly once — no + // packages/go/go/README.md duplication. + test("a top-level package file drops its dir once (no path duplication)", () => { + expect(classify("go/README.md")).toEqual({ + section: "packages", + destRel: "packages/go/README.md", + }); + }); + + // Regression 2 (apps form): group by apps- and drop the first two + // segments so the package prefix is not repeated in the rest. + test("an apps/ file groups by apps- and drops two segments", () => { + expect(classify("apps/eng-docs/README.md")).toEqual({ + section: "packages", + destRel: "packages/apps-eng-docs/README.md", + }); + }); + + test("a deeper apps/ file keeps the path below the package", () => { + expect(classify("apps/web/docs/guide.md")).toEqual({ + section: "packages", + destRel: "packages/apps-web/docs/guide.md", + }); + }); + + // Regression: a root single-segment file (empty `rest`) must not leave a + // trailing slash in destRel — that defeats routeSlug's `.md$` strip and yields + // a 404 route. `SECURITY.md` → `packages/security`, not `packages/security.md/`. + test("a root single-segment file has no trailing slash (route resolves)", () => { + expect(classify("SECURITY.md")).toEqual({ + section: "packages", + destRel: "packages/SECURITY.md", + }); + expect(routeSlug(classify("SECURITY.md").destRel)).toBe( + "/packages/security", + ); + }); +}); + +// -- parseExclusions ---------------------------------------------------------- +// Reads the canonical .markdownlint-cli2.jsonc `ignores` (single source of +// truth), tolerating // line comments, and always appends the one glob the +// gather owns: generated outputs/. + +describe("parseExclusions", () => { + test("strips // line comments, keeps ignores, appends the owned glob", () => { + const config = [ + "{", + "\t// scopes the linter to the repo", + '\t"globs": ["**/*.md"],', + "\t// exclusions below", + '\t"ignores": [', + '\t\t"forks/*/**",', + '\t\t"config/prompts/**",', + '\t\t"config/agents/**"', + "\t]", + "}", + ].join("\n"); + expect(parseExclusions(config)).toEqual([ + "forks/*/**", + "config/prompts/**", + "config/agents/**", + "**/outputs/**", + ]); + }); + + test("still yields the owned glob when the config has no ignores", () => { + expect(parseExclusions('{\n\t"globs": ["**/*.md"]\n}')).toEqual([ + "**/outputs/**", + ]); + }); +}); + +// -- isExcluded --------------------------------------------------------------- +// A path is dropped from the gather when it is the docsite's own tree (so it +// never mirrors itself) or matches an exclusion glob. Globs are path-segment +// anchored: ** any depth, **/ any depth incl zero, * one segment. + +describe("isExcluded", () => { + test("excludes the docsite's own tree even with no exclusion globs", () => { + expect(isExcluded("apps/eng-docs/src/whatever.md", [])).toBe(true); + }); + + test("does not exclude an ordinary package doc", () => { + expect( + isExcluded("go/README.md", [ + "forks/*/**", + "config/prompts/**", + "**/outputs/**", + ]), + ).toBe(false); + }); + + test("excludes a vendored fork subtree via forks/*/**", () => { + expect(isExcluded("forks/oh-my-pi/README.md", ["forks/*/**"])).toBe(true); + expect(isExcluded("forks/oh-my-pi/src/deep/x.md", ["forks/*/**"])).toBe( + true, + ); + }); + + test("keeps the first-party forks/README.md (forks/*/** does not match it)", () => { + // The glob requires a fork dir between forks/ and the file; forks/README.md + // has none, so it stays linted and gathered. + expect(isExcluded("forks/README.md", ["forks/*/**"])).toBe(false); + }); + + test("excludes any outputs/ directory at any depth via **/outputs/**", () => { + expect(isExcluded("apps/foo/outputs/report.md", ["**/outputs/**"])).toBe( + true, + ); + }); + + test("matches **/outputs/** at depth zero (top-level outputs/)", () => { + expect(isExcluded("outputs/report.md", ["**/outputs/**"])).toBe(true); + }); + + test("**/outputs/** is segment-anchored, not a substring match", () => { + // `myoutputs` is a different directory than `outputs` — must not match. + expect(isExcluded("apps/myoutputs/report.md", ["**/outputs/**"])).toBe( + false, + ); + }); + + test("excludes the agent-context payload subtrees via config/prompts/**", () => { + expect(isExcluded("config/prompts/manager.md", ["config/prompts/**"])).toBe( + true, + ); + }); +}); + +// -- editUrlFor --------------------------------------------------------------- +// The per-page "edit this file" URL points at the true canonical source path +// on GitHub's main branch, regardless of where the page is mirrored on-site. + +describe("editUrlFor", () => { + test("builds the GitHub edit URL for a repo-relative source path", () => { + expect(editUrlFor("docs/designs/repo/compass-eng-docs/design.md")).toBe( + "https://github.com/sealedsecurity/compass/edit/main/docs/designs/repo/compass-eng-docs/design.md", + ); + }); +}); + +// -- rewriteLinks ------------------------------------------------------------- +// In-repo links are rewritten so they resolve on the site: relative links to +// gathered *.md become site routes; relative links to any other in-repo file +// become GitHub blob URLs; absolute/anchor/root-absolute/mailto are untouched. +// Anchors and queries ride along; `.`/`..` are collapsed against the source dir. + +describe("rewriteLinks", () => { + const src = "docs/designs/repo/compass-eng-docs/design.md"; + // The gather's collected file list. Only a link resolving to a member .md + // becomes a site route; everything else degrades to a GitHub source URL. + const gathered = new Set([ + "docs/specs/platform/technology-stack.md", + "docs/designs/repo/compass-eng-docs/sibling.md", + ]); + + test("rewrites a relative .md link to its site route, collapsing `..`", () => { + expect( + rewriteLinks( + "see [tech](../../../specs/platform/technology-stack.md)", + src, + gathered, + ), + ).toBe("see [tech](/specs/platform/technology-stack)"); + }); + + test("preserves a #anchor suffix on a rewritten .md route", () => { + expect( + rewriteLinks( + "see [tech](../../../specs/platform/technology-stack.md#deploy)", + src, + gathered, + ), + ).toBe("see [tech](/specs/platform/technology-stack#deploy)"); + }); + + test("preserves a ?query suffix on a rewritten .md route", () => { + expect(rewriteLinks("see [x](./sibling.md?v=1)", src, gathered)).toBe( + "see [x](/designs/repo/compass-eng-docs/sibling?v=1)", + ); + }); + + test("keeps a link title after rewriting the .md target", () => { + expect( + rewriteLinks( + 'see [tech](../../../specs/platform/technology-stack.md "Stack")', + src, + gathered, + ), + ).toBe('see [tech](/specs/platform/technology-stack "Stack")'); + }); + + test("rewrites a relative non-md link to a GitHub blob URL", () => { + expect( + rewriteLinks( + "run [deploy](./deploy.ts)", + "apps/eng-docs/scripts/notes.md", + gathered, + ), + ).toBe( + "run [deploy](https://github.com/sealedsecurity/compass/blob/main/apps/eng-docs/scripts/deploy.ts)", + ); + }); + + test("collapses `..` when resolving a non-md blob target", () => { + expect( + rewriteLinks("[u](../shared/util.ts)", "ci/scripts/build.md", gathered), + ).toBe( + "[u](https://github.com/sealedsecurity/compass/blob/main/ci/shared/util.ts)", + ); + }); + + test("leaves absolute, protocol-relative, anchor, root-absolute and mailto links untouched", () => { + const links = + "[abs](https://example.com/a) " + + "[proto](//cdn.example.com/x.md) " + + "[anchor](#section) " + + "[root](/already/routed) " + + "[mail](mailto:team@example.com)"; + expect(rewriteLinks(links, src, gathered)).toBe(links); + }); + + // -- the gathered-set guard -------------------------------------------------- + // An in-repo .md that was NOT gathered has no site route, so it must degrade + // to its visible GitHub source instead of a silent 404. Membership in + // `gathered` — not the .md extension — is what decides route vs blob. + + test("degrades an ungathered .md link to its GitHub blob source", () => { + // infrastructure.md resolves in-tree but is absent from `gathered`. + expect( + rewriteLinks( + "see [infra](./infrastructure.md)", + "docs/specs/platform/technology-stack.md", + gathered, + ), + ).toBe( + "see [infra](https://github.com/sealedsecurity/compass/blob/main/docs/specs/platform/infrastructure.md)", + ); + }); + + test("carries #anchor and ?query onto an ungathered .md blob fallback", () => { + expect( + rewriteLinks( + "see [infra](./infrastructure.md#deploy)", + "docs/specs/platform/technology-stack.md", + gathered, + ), + ).toBe( + "see [infra](https://github.com/sealedsecurity/compass/blob/main/docs/specs/platform/infrastructure.md#deploy)", + ); + expect( + rewriteLinks( + "see [infra](./infrastructure.md?v=1)", + "docs/specs/platform/technology-stack.md", + gathered, + ), + ).toBe( + "see [infra](https://github.com/sealedsecurity/compass/blob/main/docs/specs/platform/infrastructure.md?v=1)", + ); + }); + + test("discriminates directory (tree) from file (blob) by the trailing slash", () => { + const from = "docs/README.md"; + // Same base path: the trailing slash — not the extension — picks tree. + expect(rewriteLinks("[d](../tools/docs-publish/)", from, gathered)).toBe( + "[d](https://github.com/sealedsecurity/compass/tree/main/tools/docs-publish)", + ); + expect(rewriteLinks("[f](../tools/docs-publish)", from, gathered)).toBe( + "[f](https://github.com/sealedsecurity/compass/blob/main/tools/docs-publish)", + ); + }); + + test("membership in `gathered` — not the .md extension — flips route vs blob", () => { + const link = "see [infra](./infrastructure.md)"; + const from = "docs/specs/platform/technology-stack.md"; + const target = "docs/specs/platform/infrastructure.md"; + // Identical link + source: routed only when the target is gathered. + expect(rewriteLinks(link, from, new Set([target]))).toBe( + "see [infra](/specs/platform/infrastructure)", + ); + expect(rewriteLinks(link, from, new Set())).toBe( + "see [infra](https://github.com/sealedsecurity/compass/blob/main/docs/specs/platform/infrastructure.md)", + ); + }); +}); + +// -- transform (whole-file pipeline) ----------------------------------------- +// The composed transform: inject the derived title + per-page editUrl as +// frontmatter, drop the now-duplicated body H1, and rewrite in-repo body links. + +describe("transform", () => { + // The gather's collected list; the body-links test asserts one member routes. + const gathered = new Set(["docs/specs/platform/technology-stack.md"]); + + test("prepends title + editUrl frontmatter and strips the body H1", () => { + const src = "# Observability\n\nThe platform...\n"; + expect( + transform(src, "docs/specs/platform/observability.md", gathered), + ).toBe( + '---\ntitle: "Observability"\neditUrl: "https://github.com/sealedsecurity/compass/edit/main/docs/specs/platform/observability.md"\n---\nThe platform...\n', + ); + }); + + test("uses the basename fallback and injects no-op strip when no H1", () => { + const src = "prose with no heading\n"; + expect(transform(src, "docs/specs/tools/README.md", gathered)).toBe( + '---\ntitle: "README"\neditUrl: "https://github.com/sealedsecurity/compass/edit/main/docs/specs/tools/README.md"\n---\nprose with no heading\n', + ); + }); + + test("preserves a title with a colon end-to-end", () => { + const src = "# Docs consumption: Notion as the read layer\n\nWhy...\n"; + expect( + transform(src, "docs/designs/platform/docs-consumption.md", gathered), + ).toBe( + '---\ntitle: "Docs consumption: Notion as the read layer"\neditUrl: "https://github.com/sealedsecurity/compass/edit/main/docs/designs/platform/docs-consumption.md"\n---\nWhy...\n', + ); + }); + + test("a doc opening with a fenced `#` keeps the fence and strips the real H1", () => { + const src = "```sh\n# example\n```\n\n# Setup\n\nSteps...\n"; + expect(transform(src, "docs/specs/tools/setup.md", gathered)).toBe( + '---\ntitle: "Setup"\neditUrl: "https://github.com/sealedsecurity/compass/edit/main/docs/specs/tools/setup.md"\n---\n```sh\n# example\n```\n\nSteps...\n', + ); + }); + + test("a doc opening with a tilde-fenced `#` keeps the fence and strips the real H1", () => { + const src = "~~~sh\n# example\n~~~\n\n# Setup\n\nSteps...\n"; + expect(transform(src, "docs/specs/tools/setup.md", gathered)).toBe( + '---\ntitle: "Setup"\neditUrl: "https://github.com/sealedsecurity/compass/edit/main/docs/specs/tools/setup.md"\n---\n~~~sh\n# example\n~~~\n\nSteps...\n', + ); + }); + + test("rewrites body links while injecting the editUrl (md→route, code→blob)", () => { + const src = + "# Docsite\n\nSee [tech](../../specs/platform/technology-stack.md) and [deploy](./deploy.ts).\n"; + expect(transform(src, "docs/designs/platform/docsite.md", gathered)).toBe( + '---\ntitle: "Docsite"\neditUrl: "https://github.com/sealedsecurity/compass/edit/main/docs/designs/platform/docsite.md"\n---\nSee [tech](/specs/platform/technology-stack) and [deploy](https://github.com/sealedsecurity/compass/blob/main/docs/designs/platform/deploy.ts).\n', + ); + }); +}); + +// -- routeSlug ---------------------------------------------------------------- +// The Starlight route for a gathered doc: its content-relative path minus the +// .md extension, lowercased (Starlight lowercases slugs). Drives the landing +// page's links so they point at pages that exist. + +describe("routeSlug", () => { + test("maps a content-relative md path to a leading-slash route", () => { + expect(routeSlug("designs/repo/compass-eng-docs/design.md")).toBe( + "/designs/repo/compass-eng-docs/design", + ); + }); + + test("lowercases the route (Starlight lowercases slugs)", () => { + expect(routeSlug("contributing/README.md")).toBe("/contributing/readme"); + }); + + // Astro content collections slugify EACH path segment through github-slugger + // (astro/dist/content/utils.js:271-272 — split on path.sep, map githubSlug, + // join). github-slugger strips the `.`, so a directory like `compass-0.6` + // becomes `compass-06` on the deployed site. routeSlug must match, or the + // landing-page link 404s. + test("slugifies a dotted directory segment like Astro (strips the dot)", () => { + expect(routeSlug("designs/product/compass-0.6/design.md")).toBe( + "/designs/product/compass-06/design", + ); + }); + + test("slugifies the compass-0.5-server dotted segment", () => { + expect(routeSlug("designs/product/compass-0.5-server/design.md")).toBe( + "/designs/product/compass-05-server/design", + ); + }); + + test("slugifies the compass-0.4 dotted segment", () => { + expect(routeSlug("designs/product/compass-0.4/design.md")).toBe( + "/designs/product/compass-04/design", + ); + }); + + test("leaves a non-dotted segment unchanged", () => { + expect(routeSlug("packages/security.md")).toBe("/packages/security"); + }); + + // Astro drops a trailing `/index` segment so `.../guide/index.md` renders at + // `.../guide`, not `.../guide/index` (astro/dist/content/utils.js:272). + test("strips a trailing /index segment", () => { + expect(routeSlug("designs/platform/guide/index.md")).toBe( + "/designs/platform/guide", + ); + }); + + // The strip is anchored to a full `/index` segment boundary, NOT a substring: + // mis-anchoring it (e.g. `/index$` without the `/`) would eat the tail of a + // real `myindex` page and 404 that route on the deployed site. + test("does not strip a segment merely ending in index", () => { + expect(routeSlug("designs/platform/myindex.md")).toBe( + "/designs/platform/myindex", + ); + }); +}); + +// -- buildIndex --------------------------------------------------------------- +// Starlight has no route for the site root or the autogenerated group dirs, so +// the gather emits a splash landing linking to the first page of each populated +// section. Links follow SECTIONS order and use SECTIONS labels; an empty +// section is omitted so no link 404s. `DomainEntry.domain` is a section key. + +describe("buildIndex", () => { + // Entries given out of SECTIONS order to prove the output is re-ordered. + const entries = [ + { domain: "contributing", route: "/contributing/readme", title: "Readme" }, + { + domain: "architecture", + route: "/architecture/overview", + title: "Overview", + }, + { + domain: "designs", + route: "/designs/repo/compass-eng-docs/design", + title: "Docsite", + }, + ]; + + test("emits splash frontmatter with a title", () => { + const md = buildIndex(entries); + expect(md).toContain('title: "Compass Engineering Docs"'); + expect(md).toContain("template: splash"); + }); + + test("links each section to its real page route with the SECTIONS label", () => { + const md = buildIndex(entries); + expect(md).toContain( + "- [Designs](/designs/repo/compass-eng-docs/design) — Docsite", + ); + expect(md).toContain("- [Architecture](/architecture/overview) — Overview"); + expect(md).toContain("- [Contributing](/contributing/readme) — Readme"); + }); + + test("orders the links by SECTIONS, not by entry order", () => { + const md = buildIndex(entries); + expect(md.indexOf("[Designs]")).toBeLessThan(md.indexOf("[Architecture]")); + expect(md.indexOf("[Architecture]")).toBeLessThan( + md.indexOf("[Contributing]"), + ); + }); + + test("omits a section with no gathered page", () => { + // specs/packages empty this run → not linked (would 404). + const md = buildIndex(entries); + expect(md).not.toContain("[Specs]"); + expect(md).not.toContain("[Packages]"); + }); +}); + +// -- buildSidebar ------------------------------------------------------------- +// The generated sidebar module: one autogenerate group per populated section, +// in SECTIONS order, with absent sections omitted. Emitted as a TS module, so +// we parse the `export const sidebar = [...]` array back out and assert on it. + +interface SidebarGroup { + label: string; + items: Array<{ autogenerate: { directory: string } }>; +} + +function parseSidebar(module: string): SidebarGroup[] { + const match = module.match(/export const sidebar = ([\s\S]*);\n$/); + if (!match) throw new Error("sidebar export not found in generated module"); + return JSON.parse(match[1]) as SidebarGroup[]; +} + +describe("buildSidebar", () => { + test("emits groups in SECTIONS order regardless of input order, with SECTIONS labels", () => { + // All five keys, shuffled — output must be the canonical order. + const groups = parseSidebar( + buildSidebar([ + "contributing", + "packages", + "architecture", + "specs", + "designs", + ]), + ); + expect(groups.map((g) => g.label)).toEqual([ + "Designs", + "Specs", + "Architecture", + "Packages", + "Contributing", + ]); + expect(groups.map((g) => g.items[0].autogenerate.directory)).toEqual([ + "designs", + "specs", + "architecture", + "packages", + "contributing", + ]); + }); + + test("omits sections that are not present", () => { + const groups = parseSidebar(buildSidebar(["packages", "designs"])); + expect(groups.map((g) => g.label)).toEqual(["Designs", "Packages"]); + expect(groups.map((g) => g.items[0].autogenerate.directory)).toEqual([ + "designs", + "packages", + ]); + }); +}); diff --git a/apps/eng-docs/scripts/gather.ts b/apps/eng-docs/scripts/gather.ts index 6be25942..a3c3dae7 100644 --- a/apps/eng-docs/scripts/gather.ts +++ b/apps/eng-docs/scripts/gather.ts @@ -1,54 +1,478 @@ -// T1 placeholder (SEA-1763) — replaced by the real corpus gather in T2 (SEA-1764). +// Gather the reviewed monorepo markdown into the Starlight content collection +// (SEA-1764). The canonical sources carry no frontmatter and open with a `# H1`; +// Starlight's docsSchema requires a `title`. This mirrors each source into +// apps/eng-docs/src/content/docs/ with a derived `title:` block, the now- +// duplicated body H1 removed, and in-repo links rewritten to site routes / +// GitHub blobs. // -// The real gather mirrors the compass monorepo's reviewed markdown into the -// gitignored src/content/docs/ tree and emits src/sidebar.generated.ts, exactly -// as sealed/apps/docs/scripts/gather.ts does. That port (compass taxonomy, -// link rewriting, frontmatter injection) is T2's scope, not T1's. +// Coverage is the whole monorepo: every tracked `*.md` except the canonical +// top-level exclusion set (`.markdownlint-cli2.jsonc` `ignores`, the single +// source of truth) and generated `outputs/`. Each file is classified into one +// nav SECTION (Designs, Specs, Architecture, Packages, Contributing); the +// section drives both its dest path under the content root and the generated +// sidebar. This file stays the single gather entry point and emits +// `src/sidebar.generated.ts` for astro.config.mjs to import. // -// For the scaffold, this stub writes only what a green `astro build` / `astro -// check` needs on an EMPTY corpus: -// - a single placeholder index page, so Starlight's `docs` content collection -// is non-empty (Starlight errors on an empty collection), and -// - an empty `src/sidebar.generated.ts` exporting `sidebar = []`, which -// astro.config.mjs imports. -// It maps NO real docs. When T2 lands the real gather, delete this file's body -// and port the sealed implementation. - -import { mkdir, writeFile } from "node:fs/promises"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; +// The pipeline splits pure construction (the exported functions below, unit- +// tested in gather.test.ts) from execution (main(), a thin fs runner). + +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { basename, dirname, join } from "node:path"; +import { Glob } from "bun"; +import { slug as githubSlug } from "github-slugger"; + +/** The GitHub repo the docsite renders — for edit links + code-file link rewrites. */ +const REPO_SLUG = "sealedsecurity/compass"; + +/** + * The `docs/` subtrees that map to a nav section by their directory name. Each + * renders under its own section verbatim (the on-disk layout is the taxonomy). + */ +const DOMAINS = ["designs", "specs", "architecture"] as const; + +/** + * A nav section: its sidebar label and the content-root directory its pages + * live under. Order here is the sidebar order. + */ +export interface Section { + readonly key: string; + readonly label: string; +} + +/** The nav taxonomy, in sidebar order. `key` is the content-root subdirectory. */ +export const SECTIONS: readonly Section[] = [ + { key: "designs", label: "Designs" }, + { key: "specs", label: "Specs" }, + { key: "architecture", label: "Architecture" }, + { key: "packages", label: "Packages" }, + { key: "contributing", label: "Contributing" }, +] as const; + +/** Repo-root files that are contributor-facing conventions, not package docs. */ +const CONTRIBUTING_FILES = new Set([ + "README.md", + "AGENTS.md", + "CONTRIBUTING.md", + "forks/README.md", +]); + +/** A classified source: which section it belongs to and where it renders. */ +export interface Classified { + /** Section key (a `SECTIONS` entry). */ + section: string; + /** Path under the content root (`src/content/docs/`), no leading slash. */ + destRel: string; +} + +/** + * The package a non-`docs/` source belongs to, split into a grouping `id` and + * the path `rest` within that package. `apps/` is a two-level unit (group by + * `apps-`); a file directly under a top dir (e.g. `ci/README.md`) groups by + * that dir. Drives `packages//` so each package's docs sit together + * without repeating the package prefix in the path. + */ +function packagePath(sourcePath: string): { id: string; rest: string } { + const parts = sourcePath.split("/"); + if (parts[0] === "apps" && parts.length > 2) { + return { id: `${parts[0]}-${parts[1]}`, rest: parts.slice(2).join("/") }; + } + return { id: parts[0], rest: parts.slice(1).join("/") }; +} + +/** + * Classify a repo-relative markdown path into its nav section + dest path. + * The single source of truth for where every gathered file lands — both the + * on-disk mirror and (via the dest path) its site route. + * + * - `docs//…` keeps its natural layout under the matching section. + * - Root README/AGENTS/CONTRIBUTING/forks-README → `contributing/`. + * - everything else (per-package README/AGENTS) → `packages//…`. + */ +export function classify(sourcePath: string): Classified { + const docsMatch = sourcePath.match(/^docs\/([^/]+)\/(.+)$/); + if (docsMatch) { + const [, domain, rest] = docsMatch; + if ((DOMAINS as readonly string[]).includes(domain)) { + return { section: domain, destRel: `${domain}/${rest}` }; + } + } + if (CONTRIBUTING_FILES.has(sourcePath)) { + // Disambiguate by source location so root README.md and forks/README.md + // (both basename README.md) don't collide under contributing/. + const slug = sourcePath.replace(/^\./, "").replace(/\//g, "-"); + return { section: "contributing", destRel: `contributing/${slug}` }; + } + const pkg = packagePath(sourcePath); + // A root single-segment file (e.g. `SECURITY.md`) has an empty `rest`; joining + // it as `packages/${id}/${rest}` would leave a trailing slash, which defeats + // routeSlug's `.md$` strip and yields a 404 route (`/packages/security.md/`). + // Omit the empty segment so both the on-disk mirror and the route are clean. + return { + section: "packages", + destRel: pkg.rest ? `packages/${pkg.id}/${pkg.rest}` : `packages/${pkg.id}`, + }; +} + +/** + * The top-level exclusion globs, read from the canonical `.markdownlint-cli2.jsonc` + * `ignores` list (the single source of truth) plus the one the gather always + * adds: generated `outputs/`. Declared once there, not duplicated here — a new + * excluded dir is added in that config. + */ +export function parseExclusions(markdownlintConfig: string): string[] { + // The file is JSONC; strip // line comments before parsing. + const stripped = markdownlintConfig.replace(/^\s*\/\/.*$/gm, ""); + const parsed = JSON.parse(stripped) as { ignores?: string[] }; + const ignores = parsed.ignores ?? []; + return [...ignores, "**/outputs/**"]; +} + +// ── Pure construction ─────────────────────────────────────────────────────── + +/** + * The line index of the first markdown `# H1`, or `-1` when there is none. + * `#` characters inside fenced code blocks are not headings and are skipped — + * the single source of truth for "where is the H1" shared by `extractTitle` + * (reads its text) and `stripFirstH1` (drops the line), so the two can never + * disagree about fence context. Both CommonMark fence markers are recognized + * (```` ``` ```` and `~~~`); a fence closes only on the same marker at a length + * >= the opener, so a `#` inside either kind of block is never read as the H1. + */ +function firstH1Index(lines: readonly string[]): number { + let fence: { marker: "`" | "~"; length: number } | undefined; + for (let i = 0; i < lines.length; i++) { + const trimmed = lines[i].trim(); + const fenceMatch = trimmed.match(/^(`{3,}|~{3,})/); + if (fence) { + if ( + fenceMatch && + fenceMatch[1][0] === fence.marker && + fenceMatch[1].length >= fence.length + ) { + fence = undefined; + } + continue; + } + if (fenceMatch) { + fence = { + marker: fenceMatch[1][0] as "`" | "~", + length: fenceMatch[1].length, + }; + continue; + } + if (/^#\s+.+$/.test(trimmed)) return i; + } + return -1; +} + +/** + * The title for a source: the text of its first `# H1`, or — when a file has + * none — its basename without extension. `#` characters inside fenced code + * blocks are not headings and are skipped. + */ +export function extractTitle(source: string, sourcePath = ""): string { + const lines = source.split("\n"); + const i = firstH1Index(lines); + if (i !== -1) return lines[i].trim().replace(/^#\s+/, "").trim(); + return basename(sourcePath).replace(/\.[^.]+$/, ""); +} + +/** + * A YAML frontmatter block carrying the title (and, when given, a per-page + * `editUrl`). The title is always a double-quoted scalar so YAML-significant + * characters in it (`:`, `#`, quotes) are inert; backslashes and double quotes + * are escaped. With no `editUrl` the block is title-only (the docs/ case, where + * the global editLink base already resolves correctly). + */ +export function buildFrontmatter(title: string, editUrl?: string): string { + const escaped = title.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); + const editLine = editUrl ? `editUrl: "${editUrl}"\n` : ""; + return `---\ntitle: "${escaped}"\n${editLine}---\n`; +} + +/** The canonical GitHub "edit this file" URL for a repo-relative source path. */ +export function editUrlFor(sourcePath: string): string { + return `https://github.com/${REPO_SLUG}/edit/main/${sourcePath}`; +} + +/** + * Drop the first `# H1` line (Starlight renders `title` as the page h1, so the + * body copy would otherwise double it) plus a single blank line immediately + * after it. A `#` inside a leading fenced code block is not the H1 and is left + * intact; later H1s and all other content are untouched. + */ +export function stripFirstH1(source: string): string { + const lines = source.split("\n"); + const i = firstH1Index(lines); + if (i === -1) return source; + const drop = lines[i + 1] === "" ? 2 : 1; + lines.splice(i, drop); + return lines.join("\n"); +} + +export function destRelPath(sourcePath: string): string { + return sourcePath.replace(/^docs\//, ""); +} + +/** File extensions that render as a site page — links to these become routes. */ +const MARKDOWN_EXT = /\.md$/i; + +/** + * Rewrite in-repo links in a gathered file so they resolve on the site: + * + * - a relative link to another **gathered** `*.md` → its site route (via + * `classify` + `routeSlug`), so cross-doc links stay internal; + * - a relative link to any other in-repo target → its canonical GitHub URL: + * `/tree/main/…` for a directory (trailing slash), `/blob/main/…` otherwise + * (code, image, config — and any `.md` that was *not* gathered, e.g. an + * excluded tree or a stale/moved path). Routing off the actual gathered set + * means a link to an ungathered file degrades to its visible source on GitHub + * instead of a silent 404 on a route the site never generated; + * - absolute URLs (`http(s):`, `//`), anchors (`#…`), and mailto are untouched. + * + * `sourcePath` is the repo-relative path of the file being transformed, used to + * resolve relative targets against the source's directory. `gathered` is the set + * of repo-relative paths that became site pages (the gather's own file list). + */ +export function rewriteLinks( + source: string, + sourcePath: string, + gathered: ReadonlySet, +): string { + const srcDir = dirname(sourcePath); + // Markdown inline links: [text](target) and [text](target "title"). + return source.replace( + /(\]\()([^)\s]+)(\s+"[^"]*")?(\))/g, + (match, open, target, title, close) => { + if ( + /^(?:[a-z][a-z0-9+.-]*:|\/\/|#|mailto:)/i.test(target) || + target.startsWith("/") + ) { + return match; + } + // Split off any anchor / query so it rides along to the new target. + const hashIdx = target.search(/[#?]/); + const path = hashIdx === -1 ? target : target.slice(0, hashIdx); + const suffix = hashIdx === -1 ? "" : target.slice(hashIdx); + const resolved = normalizeRepoPath(join(srcDir, path)); + let rewritten: string; + if (MARKDOWN_EXT.test(resolved) && gathered.has(resolved)) { + rewritten = routeSlug(classify(resolved).destRel) + suffix; + } else { + // Directory links (trailing slash) need GitHub's `tree` view; every + // other target — files, and `.md` that was never gathered — is a `blob`. + const kind = path.endsWith("/") ? "tree" : "blob"; + rewritten = `https://github.com/${REPO_SLUG}/${kind}/main/${resolved}${suffix}`; + } + return `${open}${rewritten}${title ?? ""}${close}`; + }, + ); +} + +/** Collapse `.`/`..` segments in a POSIX repo path (no leading slash). */ +function normalizeRepoPath(p: string): string { + const out: string[] = []; + for (const seg of p.split("/")) { + if (seg === "" || seg === ".") continue; + if (seg === "..") out.pop(); + else out.push(seg); + } + return out.join("/"); +} + +/** + * Whole-file transform: inject the derived title + per-page GitHub `editUrl`, + * strip the duplicated H1, and rewrite in-repo links to site routes / GitHub + * blobs. The `editUrl` points at the true canonical source (`sourcePath`), so a + * gathered file mirrored under a different content-root path still edits the + * right file. + */ +export function transform( + source: string, + sourcePath: string, + gathered: ReadonlySet, +): string { + return ( + buildFrontmatter(extractTitle(source, sourcePath), editUrlFor(sourcePath)) + + rewriteLinks(stripFirstH1(source), sourcePath, gathered) + ); +} + +/** A gathered doc's landing-page entry: its section, Starlight route, and title. */ +export interface DomainEntry { + /** Section key (a `SECTIONS` entry). */ + domain: string; + route: string; + title: string; +} + +/** + * The Starlight route for a content-relative markdown path. Astro content + * collections slug a page by dropping the `.md`, slugifying EACH path segment + * with `github-slugger` (lowercase, strip `.` and most punctuation, spaces→`-`), + * then dropping a trailing `/index` (astro/dist/content/utils.js:271-272). We + * reuse Astro's own slugger so a route can never drift from where the page + * actually renders — a plain `.toLowerCase()` kept dots (`compass-0.6`), which + * 404s against Astro's `compass-06`. Used for the landing-page links, in-page + * cross-doc link rewrites, and the PR preview comment's changed-page deep-links — + * all of which must resolve on the deployed site. `apps/eng-docs/package.json`'s + * `github-slugger` must stay on the same major as astro's transitive copy, or + * the two slug differently and routes drift again. + */ +export function routeSlug(contentRelPath: string): string { + return `/${contentRelPath + .replace(/\.md$/, "") + .split("/") + .map((segment) => githubSlug(segment)) + .join("/") + .replace(/\/index$/, "")}`; +} + +/** + * The root landing page (`src/content/docs/index.md`). Starlight generates no + * route for the site root or the autogenerated group dirs, so without this the + * preview comment's root URL 404s. A `splash` page linking to the first page of + * each populated section (a route guaranteed to exist); an empty section is + * omitted so no link 404s. + */ +export function buildIndex(entries: readonly DomainEntry[]): string { + const bySection = new Map(entries.map((e) => [e.domain, e])); + const links = SECTIONS.filter((s) => bySection.has(s.key)) + .map((s) => { + const e = bySection.get(s.key) as DomainEntry; + return `- [${s.label}](${e.route}) — ${e.title}`; + }) + .join("\n"); + return ( + "---\n" + + 'title: "Compass Engineering Docs"\n' + + 'description: "Public engineering docs for the compass monorepo."\n' + + "template: splash\n" + + "---\n\n" + + "The compass monorepo's reviewed documentation. Browse by section:\n\n" + + `${links}\n` + ); +} + +/** + * The generated Starlight sidebar (`src/sidebar.generated.ts`), imported by + * astro.config.mjs. Each populated section becomes a group that autogenerates + * from its content-root directory; an empty section is omitted so Starlight + * never errors on a missing directory. Emitted as a typed module so the astro + * config stays declarative. + */ +export function buildSidebar(sections: readonly string[]): string { + const present = new Set(sections); + const groups = SECTIONS.filter((s) => present.has(s.key)).map((s) => ({ + label: s.label, + items: [{ autogenerate: { directory: s.key } }], + })); + return ( + "// Generated by scripts/gather.ts — do not edit.\n" + + "// The monorepo docsite sidebar, one group per populated nav section\n" + + "// (SEA-1764). Regenerated on every gather; gitignored like the content mirror.\n" + + `export const sidebar = ${JSON.stringify(groups, null, "\t")};\n` + ); +} + +// ── Execution ──────────────────────────────────────────────────────────────── + +/** + * Whether a repo-relative path is excluded from the gather, given the exclusion + * globs (from `parseExclusions`). A glob supports `**` (any depth) and `*` (one + * segment). Also excludes the docsite's own tree so it never mirrors itself. + */ +export function isExcluded( + sourcePath: string, + exclusions: readonly string[], +): boolean { + if (sourcePath.startsWith("apps/eng-docs/")) return true; + return exclusions.some((glob) => globToRegExp(glob).test(sourcePath)); +} + +/** Compile a simple `**`/`*` path glob to an anchored RegExp. */ +function globToRegExp(glob: string): RegExp { + const re = glob + .split(/(\*\*\/|\*\*|\*)/) + .map((part) => { + if (part === "**/") return "(?:.*/)?"; + if (part === "**") return ".*"; + if (part === "*") return "[^/]*"; + return part.replace(/[.+?^${}()|[\]\\]/g, "\\$&"); + }) + .join(""); + return new RegExp(`^${re}$`); +} async function main(): Promise { - // apps/eng-docs/scripts/gather.ts → app root is one level up. - const scriptDir = dirname(fileURLToPath(import.meta.url)); + // apps/eng-docs/scripts/gather.ts → repo root is three levels up. + const scriptDir = dirname(Bun.fileURLToPath(import.meta.url)); const appDir = dirname(scriptDir); + const repoRoot = dirname(dirname(appDir)); const contentRoot = join(appDir, "src", "content", "docs"); - await mkdir(contentRoot, { recursive: true }); + // Idempotent rebuild: clear the generated mirror, then repopulate. + await rm(contentRoot, { recursive: true, force: true }); - // Placeholder root landing page so the collection is non-empty and `/` has a - // route. Replaced by the real generated index + section pages in T2. - await writeFile( - join(contentRoot, "index.md"), - "---\n" + - 'title: "Compass Engineering Docs"\n' + - 'description: "Public engineering docs for the compass monorepo."\n' + - "template: splash\n" + - "---\n\n" + - "Scaffold placeholder — the reviewed documentation corpus is gathered here\n" + - "in T2 (SEA-1764).\n", + // The canonical exclusion set lives in .markdownlint-cli2.jsonc (+ the one + // the gather always adds); read it so a new excluded dir is declared once. + const exclusions = parseExclusions( + await readFile(join(repoRoot, ".markdownlint-cli2.jsonc"), "utf8"), ); - // Empty sidebar (no populated sections yet); astro.config.mjs imports this. + // All markdown under the repo, minus node_modules/build output and the + // exclusion set. Sorted for deterministic per-section "first page" + output. + const glob = new Glob("**/*.md"); + const rels: string[] = []; + for await (const rel of glob.scan({ + cwd: repoRoot, + onlyFiles: true, + dot: true, + })) { + if (rel.includes("node_modules/") || rel.includes("/dist/")) continue; + // dot:true surfaces .github (wanted) but also VCS/tooling dotdirs — skip those. + if (/(^|\/)\.(git|astro|direnv|moon|vscode|idea|pagefind)\//.test(rel)) { + continue; + } + if (isExcluded(rel, exclusions)) continue; + rels.push(rel); + } + rels.sort(); + // The set of paths that become site pages — the source of truth for whether + // a cross-doc `.md` link resolves to a route or falls back to its GitHub source. + const gathered = new Set(rels); + + let count = 0; + // First gathered page per section anchors that section's landing-page link. + const entries: DomainEntry[] = []; + const sectionsSeen = new Set(); + for (const rel of rels) { + const source = await Bun.file(join(repoRoot, rel)).text(); + const { section, destRel } = classify(rel); + const dest = join(contentRoot, destRel); + await mkdir(dirname(dest), { recursive: true }); + await writeFile(dest, transform(source, rel, gathered)); + if (!sectionsSeen.has(section)) { + sectionsSeen.add(section); + entries.push({ + domain: section, + route: routeSlug(destRel), + title: extractTitle(source, rel), + }); + } + count++; + } + + // Root landing page: Starlight has no route for `/` or the group dirs, so + // without this the preview comment's root URL 404s. + await writeFile(join(contentRoot, "index.md"), buildIndex(entries)); + // Generated sidebar for astro.config.mjs (one group per populated section). await writeFile( join(appDir, "src", "sidebar.generated.ts"), - "// Generated by scripts/gather.ts — do not edit.\n" + - "// T1 placeholder (SEA-1763): empty until the real corpus gather lands in T2.\n" + - "export const sidebar = [];\n", + buildSidebar([...sectionsSeen]), ); - console.log( - `gather (T1 stub): wrote placeholder index + empty sidebar into ${contentRoot}`, + `gather: wrote ${count} docs + index + sidebar into ${contentRoot}`, ); }