COMDOX-1733: paginate GraphQL API reference for all schema versions

scripts/generate-spectaql-md.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 // Runs SpectaQL for each GraphQL schema version, splits the monolithic output
-// into per-section chunk files, and syncs the Fragment includes in the
-// corresponding index.md.
+// into per-section chunk files, and syncs reference pages that embed each chunk
+// via a single Fragment include.
 //
 // Usage:
 //   npm run generate:graphql-api-docs           # all versions
@@ -15,9 +15,30 @@ const yaml = require('js-yaml');
 
 const ROOT = path.join(__dirname, '..');
 
-// Maximum bytes per types chunk file. Individual type entries may push a chunk
-// slightly over this limit; the boundary is always at an H3 heading.
-const MAX_TYPES_CHUNK_BYTES = 200 * 1024;
+// Types are split into fixed alphabetical ranges so each range maps to its own
+// reference page instead of byte-sized chunks that still load on one page.
+const TYPE_LETTER_RANGES = [
+  { suffix: 'a-b', letters: 'AB', navTitle: 'Types A–B', heading: 'Types A–B' },
+  { suffix: 'c-e', letters: 'CDE', navTitle: 'Types C–E', heading: 'Types C–E' },
+  { suffix: 'f-i', letters: 'FGHIJ', navTitle: 'Types F–I', heading: 'Types F–I' },
+  { suffix: 'k-p', letters: 'KLMNOP', navTitle: 'Types K–P', heading: 'Types K–P' },
+  { suffix: 'q-s', letters: 'QRS', navTitle: 'Types Q–S', heading: 'Types Q–S' },
+  { suffix: 't-z', letters: 'TUVWXYZ', navTitle: 'Types T–Z', heading: 'Types T–Z' },
+];
+
+const TYPES_HEADER = '## Types\n\n';
+
+// Normalize SpectaQL Markdown for markdownlint: collapse excess blank lines and
+// strip trailing whitespace from each line (MD009; SpectaQL often emits single
+// trailing spaces on table rows and wrapped scalar descriptions).
+function sanitizeMarkdown(content) {
+  return content
+    .split('\n')
+    .map(line => line.replace(/[ \t]+$/, ''))
+    .join('\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trimEnd() + '\n';
+}
 
 // Reads spectaql.targetDir and spectaql.targetFile from a config YAML and
 // returns the resolved absolute path of the output file.
@@ -63,62 +84,133 @@ function parseSections(content) {
   return sections;
 }
 
-// Split a block of content at H3 boundaries into chunks whose UTF-8 size does
-// not exceed maxBytes (unless a single entry is already larger).
-function chunkByH3(content, maxBytes) {
-  // Split at the start of every H3 heading using a lookahead so the delimiter
-  // stays attached to the following chunk.
-  const parts = content.split(/(?=^### )/m);
-
-  if (parts.length <= 1) return [content];
+function letterRangeFor(typeName) {
+  const letter = typeName.charAt(0).toUpperCase();
+  return TYPE_LETTER_RANGES.find(range => range.letters.includes(letter));
+}
 
-  const chunks = [];
-  let current = '';
+// Split the types section at H3 boundaries into fixed alphabetical ranges.
+function chunkByLetterRange(content) {
+  const parts = content.split(/(?=^### )/m);
+  const buckets = Object.fromEntries(TYPE_LETTER_RANGES.map(range => [range.suffix, '']));
 
   for (const part of parts) {
-    if (!current) {
-      current = part;
-    } else if (Buffer.byteLength(current + part, 'utf8') > maxBytes) {
-      chunks.push(current);
-      current = part;
-    } else {
-      current += part;
+    const h3Match = part.match(/^### (\S+)/);
+    if (!h3Match) {
+      continue;
+    }
+
+    const range = letterRangeFor(h3Match[1]);
+    if (!range) {
+      console.warn(`  warning: type "${h3Match[1]}" has no letter range`);
+      continue;
     }
+    buckets[range.suffix] += part;
   }
-  if (current) chunks.push(current);
-  return chunks;
+
+  return TYPE_LETTER_RANGES.map(range => ({
+    suffix: range.suffix,
+    content: buckets[range.suffix]
+      ? (TYPES_HEADER + buckets[range.suffix]).trimEnd() + '\n'
+      : TYPES_HEADER,
+  }));
 }
 
-// Replace the <Fragment …/> block at the end of an index.md with fresh entries.
-// fragmentsRelPath is the relative path from the index.md's directory to the
-// autogenerated includes directory (e.g. "../../includes/autogenerated").
-function updateIndexFragments(indexPath, fragmentFiles, fragmentsRelPath) {
-  if (!fs.existsSync(indexPath)) {
-    console.warn(`  warning: index.md not found at ${indexPath}`);
-    return;
-  }
+function yamlQuote(value) {
+  return JSON.stringify(value);
+}
 
-  const content = fs.readFileSync(indexPath, 'utf8');
-  const firstFragment = content.indexOf('<Fragment');
-  const base = firstFragment === -1
-    ? content.trimEnd() + '\n\n'
-    : content.slice(0, firstFragment);
+// Write or update a reference page that embeds a single generated fragment.
+function writeReferencePage(pagePath, { title, description, heading, fragmentFile, fragmentsRelPath }) {
+  const content = [
+    '---',
+    `title: ${yamlQuote(title)}`,
+    `description: ${yamlQuote(description)}`,
+    'keywords:',
+    '  - GraphQL',
+    '---',
+    '',
+    `# ${heading}`,
+    '',
+    `<Fragment src="${fragmentsRelPath}/${fragmentFile}" />`,
+    '',
+  ].join('\n');
+
+  fs.writeFileSync(pagePath, content, 'utf8');
+}
 
-  const fragmentBlock = fragmentFiles
-    .map(f => `<Fragment src="${fragmentsRelPath}/${f}" />`)
-    .join('\n\n');
+function schemaDescription(meta, sectionLabel) {
+  if (meta.version === 'saas') {
+    return `Review GraphQL ${sectionLabel} in the ${meta.productLabel} API schema.`;
+  }
+  return `Review GraphQL ${sectionLabel} in the ${meta.productLabel} ${meta.version} API schema.`;
+}
 
-  fs.writeFileSync(indexPath, base + fragmentBlock + '\n', 'utf8');
+function syncReferencePages(indexDir, schemaMeta, pageSpecs, fragmentsRelPath) {
+  const indexAbsDir = path.resolve(ROOT, indexDir);
+
+  for (const spec of pageSpecs) {
+    const title = spec.pageTitleSuffix
+      ? `${schemaMeta.titleBase}${spec.pageTitleSuffix}`
+      : schemaMeta.titleBase;
+    const description = spec.description(schemaMeta);
+    const heading = spec.heading(schemaMeta);
+    const pagePath = path.join(indexAbsDir, spec.pageFile);
+
+    writeReferencePage(pagePath, {
+      title,
+      description,
+      heading,
+      fragmentFile: spec.fragmentFile,
+      fragmentsRelPath,
+    });
+    console.log(`  updated ${path.join(indexDir, spec.pageFile)}`);
+  }
 }
 
 // To add a new schema version: add an entry here and a matching config YAML,
 // then add a generate:graphql-api-docs:X-Y-Z script to package.json.
 const schemas = [
-  { version: '2.4.9', config: 'spectaql/config.yml', indexDir: 'src/pages/reference/graphql' },
-  { version: '2.4.6', config: 'spectaql/config_2-4-6.yml', indexDir: 'src/pages/reference/graphql/2-4-6' },
-  { version: '2.4.7', config: 'spectaql/config_2-4-7.yml', indexDir: 'src/pages/reference/graphql/2-4-7' },
-  { version: '2.4.8', config: 'spectaql/config_2-4-8.yml', indexDir: 'src/pages/reference/graphql/2-4-8' },
-  { version: 'saas',  config: 'spectaql/config_saas.yml',  indexDir: 'src/pages/reference/graphql/saas'  },
+  {
+    version: '2.4.9',
+    config: 'spectaql/config.yml',
+    indexDir: 'src/pages/reference/graphql/latest',
+    titleBase: 'GraphQL API reference (2.4.9)',
+    headingBase: 'Adobe Commerce GraphQL API',
+    productLabel: 'Adobe Commerce',
+  },
+  {
+    version: '2.4.8',
+    config: 'spectaql/config_2-4-8.yml',
+    indexDir: 'src/pages/reference/graphql/2-4-8',
+    titleBase: 'GraphQL API reference (2.4.8)',
+    headingBase: 'Adobe Commerce GraphQL API (2.4.8)',
+    productLabel: 'Adobe Commerce',
+  },
+  {
+    version: '2.4.7',
+    config: 'spectaql/config_2-4-7.yml',
+    indexDir: 'src/pages/reference/graphql/2-4-7',
+    titleBase: 'GraphQL API reference (2.4.7)',
+    headingBase: 'Adobe Commerce GraphQL API (2.4.7)',
+    productLabel: 'Adobe Commerce',
+  },
+  {
+    version: '2.4.6',
+    config: 'spectaql/config_2-4-6.yml',
+    indexDir: 'src/pages/reference/graphql/2-4-6',
+    titleBase: 'GraphQL API reference (2.4.6)',
+    headingBase: 'Adobe Commerce GraphQL API (2.4.6)',
+    productLabel: 'Adobe Commerce',
+  },
+  {
+    version: 'saas',
+    config: 'spectaql/config_saas.yml',
+    indexDir: 'src/pages/reference/graphql/saas',
+    titleBase: 'GraphQL API reference (SaaS)',
+    headingBase: 'Adobe Commerce as a Cloud Service GraphQL API',
+    productLabel: 'Adobe Commerce as a Cloud Service',
+  },
 ];
 
 // Optional version filter passed as the first CLI argument.
@@ -129,7 +221,8 @@ if (filter && toRun.length === 0) {
   process.exit(1);
 }
 
-for (const { config, indexDir } of toRun) {
+for (const schema of toRun) {
+  const { config, indexDir } = schema;
   const configPath = path.resolve(ROOT, config);
   const outputFile = outputFileFor(config);
   const outputDir = path.dirname(outputFile);
@@ -171,55 +264,77 @@ for (const { config, indexDir } of toRun) {
   // Read the monolithic output, collapse excess blank lines, then delete it —
   // it will be replaced by the split chunk files below.
   const raw = fs.readFileSync(outputFile, 'utf8');
-  const content = raw.replace(/\n{3,}/g, '\n\n').trimEnd() + '\n';
+  const content = sanitizeMarkdown(raw);
   fs.unlinkSync(outputFile);
 
   const sections = parseSections(content);
-  const generatedFiles = [];
+  const pageSpecs = [];
 
   // Queries file: preamble (endpoint/header boilerplate) + queries section.
   {
-    const fileName = `${baseName}-queries.md`;
+    const fragmentFile = `${baseName}-queries.md`;
     const body = ((sections.preamble || '') + (sections.queries || '')).trimEnd() + '\n';
-    fs.writeFileSync(path.join(outputDir, fileName), body, 'utf8');
-    generatedFiles.push(fileName);
-    console.log(`  wrote ${fileName}`);
+    fs.writeFileSync(path.join(outputDir, fragmentFile), body, 'utf8');
+    console.log(`  wrote ${fragmentFile}`);
+    pageSpecs.push({
+      pageFile: 'index.md',
+      fragmentFile,
+      pageTitleSuffix: '',
+      heading: meta => meta.headingBase,
+      description: meta => schemaDescription(meta, 'queries'),
+    });
   }
 
   // Mutations section.
   if (sections.mutations) {
-    const fileName = `${baseName}-mutations.md`;
-    fs.writeFileSync(path.join(outputDir, fileName), sections.mutations.trimEnd() + '\n', 'utf8');
-    generatedFiles.push(fileName);
-    console.log(`  wrote ${fileName}`);
+    const fragmentFile = `${baseName}-mutations.md`;
+    fs.writeFileSync(path.join(outputDir, fragmentFile), sections.mutations.trimEnd() + '\n', 'utf8');
+    console.log(`  wrote ${fragmentFile}`);
+    pageSpecs.push({
+      pageFile: 'mutations.md',
+      fragmentFile,
+      pageTitleSuffix: ' – Mutations',
+      heading: () => 'Mutations',
+      description: meta => schemaDescription(meta, 'mutations'),
+    });
   }
 
   // Subscriptions section (not present in current schemas, included for
   // forward-compatibility).
   if (sections.subscriptions) {
-    const fileName = `${baseName}-subscriptions.md`;
-    fs.writeFileSync(path.join(outputDir, fileName), sections.subscriptions.trimEnd() + '\n', 'utf8');
-    generatedFiles.push(fileName);
-    console.log(`  wrote ${fileName}`);
+    const fragmentFile = `${baseName}-subscriptions.md`;
+    fs.writeFileSync(path.join(outputDir, fragmentFile), sections.subscriptions.trimEnd() + '\n', 'utf8');
+    console.log(`  wrote ${fragmentFile}`);
+    pageSpecs.push({
+      pageFile: 'subscriptions.md',
+      fragmentFile,
+      pageTitleSuffix: ' – Subscriptions',
+      heading: () => 'Subscriptions',
+      description: meta => schemaDescription(meta, 'subscriptions'),
+    });
   }
 
-  // Types section: split into ≤200 KB chunks at H3 boundaries.
+  // Types section: split into fixed alphabetical ranges.
   if (sections.types) {
-    const chunks = chunkByH3(sections.types, MAX_TYPES_CHUNK_BYTES);
-    for (let i = 0; i < chunks.length; i++) {
-      const fileName = `${baseName}-types-${i + 1}.md`;
-      fs.writeFileSync(path.join(outputDir, fileName), chunks[i].trimEnd() + '\n', 'utf8');
-      generatedFiles.push(fileName);
-      console.log(`  wrote ${fileName}`);
+    const chunks = chunkByLetterRange(sections.types);
+    for (const chunk of chunks) {
+      const range = TYPE_LETTER_RANGES.find(entry => entry.suffix === chunk.suffix);
+      const fragmentFile = `${baseName}-types-${chunk.suffix}.md`;
+      fs.writeFileSync(path.join(outputDir, fragmentFile), chunk.content, 'utf8');
+      console.log(`  wrote ${fragmentFile}`);
+      pageSpecs.push({
+        pageFile: `types-${chunk.suffix}.md`,
+        fragmentFile,
+        pageTitleSuffix: ` – ${range.navTitle}`,
+        heading: () => range.heading,
+        description: meta => schemaDescription(meta, range.navTitle.toLowerCase()),
+      });
     }
   }
 
-  // Sync the Fragment includes in the corresponding reference index page.
-  const indexPath = path.resolve(ROOT, indexDir, 'index.md');
   const autogeneratedDir = path.resolve(ROOT, 'src/pages/includes/autogenerated');
   const fragmentsRelPath = path.relative(path.resolve(ROOT, indexDir), autogeneratedDir);
-  updateIndexFragments(indexPath, generatedFiles, fragmentsRelPath);
-  console.log(`  updated ${indexDir}/index.md`);
+  syncReferencePages(indexDir, schema, pageSpecs, fragmentsRelPath);
 
-  console.log(`Done: ${generatedFiles.length} chunk files for ${baseName}\n`);
+  console.log(`Done: ${pageSpecs.length} reference pages for ${baseName}\n`);
 }

spectaql/README.md

@@ -23,20 +23,25 @@ This directory contains [SpectaQL](https://github.com/anvilco/spectaql) configur
 | `schema_saas.json` | GraphQL introspection result — Adobe Commerce as a Cloud Service. |
 | `markdown-theme/` | Custom SpectaQL Handlebars theme that outputs a plain Markdown fragment instead of HTML. |
 | `markdown-grunt-config.js` | Custom grunt task config that disables SpectaQL's HTML prettifier, which would otherwise collapse the Markdown output to a single line. |
-| `scripts/generate-spectaql-md.js` | Build script. Runs SpectaQL once per schema, splits the output into per-section chunk files, removes SpectaQL's JS/CSS asset output, and syncs the Fragment includes in the corresponding `index.md`. |
+| `scripts/generate-spectaql-md.js` | Build script. Runs SpectaQL once per schema, splits the output into per-section chunk files, removes SpectaQL's JS/CSS asset output, and syncs the reference pages that embed each chunk via a single Fragment include. |
 
 ## How it works
 
-The script reads each local introspection file, runs SpectaQL with a custom Markdown-output theme, and splits the monolithic output into smaller chunk files written to `src/pages/includes/autogenerated/`. Each generated chunk is included in the API reference page via Fragment directives that the script maintains automatically:
+The script reads each local introspection file, runs SpectaQL with a custom Markdown-output theme, and splits the monolithic output into smaller chunk files written to `src/pages/includes/autogenerated/`. Each chunk is embedded in its own reference page via a single Fragment directive that the script maintains automatically:
 
 ```html
+<!-- src/pages/reference/graphql/latest/index.md -->
 <Fragment src="../../../includes/autogenerated/graphql-api-2-4-9-queries.md" />
+
+<!-- src/pages/reference/graphql/latest/mutations.md -->
 <Fragment src="../../../includes/autogenerated/graphql-api-2-4-9-mutations.md" />
-<Fragment src="../../../includes/autogenerated/graphql-api-2-4-9-types-1.md" />
+
+<!-- src/pages/reference/graphql/latest/types-a-b.md -->
+<Fragment src="../../../includes/autogenerated/graphql-api-2-4-9-types-a-b.md" />
 …
 ```
 
-The deployment system imposes a size limit on individual Markdown files, so the types section is split into multiple chunks of at most ~200 KB each (split at type boundaries, never mid-entry).
+The deployment system imposes a size limit on individual Markdown files, and loading every chunk on one page defeats the purpose of splitting. Types are therefore split into fixed alphabetical ranges (`types-a-b`, `types-c-e`, `types-f-i`, `types-k-p`, `types-q-s`, `types-t-z`), each mapped to its own reference page. Queries and mutations are separate pages as well.
 
 ## Generate the API reference
 
@@ -64,8 +69,8 @@ What happens for each schema:
 1. The custom `markdown-grunt-config.js` bypasses the HTML prettifier.
 1. SpectaQL's JS/CSS asset directories are removed from the output directory.
 1. The document is split by H2 section into `*-queries.md`, `*-mutations.md`, and (if present) `*-subscriptions.md`.
-1. The types section is further split into `*-types-1.md`, `*-types-2.md`, … at H3 boundaries so each chunk stays under ~200 KB.
-1. The `<Fragment>` block in the corresponding `index.md` under `src/pages/reference/graphql/` is rewritten to reference exactly the generated chunk files.
+1. The types section is split into fixed alphabetical ranges: `*-types-a-b.md`, `*-types-c-e.md`, `*-types-f-i.md`, `*-types-k-p.md`, `*-types-q-s.md`, and `*-types-t-z.md` (split at H3 boundaries).
+1. The script writes or updates separate reference pages under `src/pages/reference/graphql/` (`index.md`, `mutations.md`, and `types-*.md`) so each page embeds exactly one chunk file.
 
 ## Update the API reference
 
@@ -78,8 +83,8 @@ If a schema changes:
    npm run generate:graphql-api-docs:X.Y.Z
    ```
 
-1. Review the updated chunk files in `src/pages/includes/autogenerated/` and the updated `index.md` in its corresponding `src/pages/reference/graphql/` directory.
-1. Commit all changed files (chunk files + `index.md` if the number of types chunks changed).
+1. Review the updated chunk files in `src/pages/includes/autogenerated/` and the updated reference pages in the corresponding `src/pages/reference/graphql/` directory.
+1. Commit all changed files (chunk files + reference pages if the page layout changed).
 1. After the PR is approved and merged, the updated reference is published automatically via EDS.
 
 ## Add a new schema version
@@ -88,7 +93,8 @@ If a schema changes:
 1. Copy `config_2-4-8.yml` to `config_X.Y.Z.yml` and update `introspectionFile`, `targetFile`, `version`, and `title`.
 1. Add an entry (with `version`, `config`, and `indexDir` fields) to the `schemas` array in `scripts/generate-spectaql-md.js`.
 1. Add a `generate:graphql-api-docs:X.Y.Z` script to `package.json`.
-1. Create `src/pages/reference/graphql/X-Y-Z/index.md` with the appropriate frontmatter, Edition badge, and heading (no Fragment lines needed — the script writes them on first run).
+1. Create `src/pages/reference/graphql/X-Y-Z/index.md` with the appropriate frontmatter and heading (no Fragment lines needed — the script writes the reference pages on first run).
+1. Add the version's sub-pages to `src/pages/config.md` (queries, mutations, and the six types ranges).
 1. Run `npm run generate:graphql-api-docs:X.Y.Z` and commit the result.
 
 ## Notes