From 7f14acb3bad42256d01db2068b60cdd9ffd9b831 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 6 Oct 2026 15:50:20 -0600
Subject: [PATCH 1/4] ci/duplicates: Add duplicated content PR check

Each fact should have one home in the docs, with other pages linking to
it; copies drift out of date the moment the original changes. This adds
a non-blocking PR check that comments the passages a PR adds which say
the same thing as a passage elsewhere under `docs/`, paraphrased or
verbatim, plus an inline comment on each added passage pointing at its
duplicates.

`dev/check-duplicates.mjs` splits every page into passages (paragraphs,
list items, tables, fenced code blocks), normalizes them, and compares
word trigrams by Jaccard similarity. Auto-generated pages and
SCHEMA_SYNC blocks are skipped. Like the links and redirects checks, CI
records the findings on the merge base and reports only the ones the PR
adds; main has hundreds of pre-existing findings.

AGENTS.md gains the "One home per fact" rules so the duplication does
not grow back.

Amp-Thread-ID: https://ampcode.com/threads/T-01a11326-478d-72cf-bbb7-587d16b378e9
Co-authored-by: Amp <amp@ampcode.com>
---
 .github/workflows/check-duplicates.yml | 119 ++++++
 AGENTS.md                              |  29 +-
 cspell-allow-list.txt                  |   1 +
 dev/check-duplicates.mjs               | 523 +++++++++++++++++++++++++
 4 files changed, 666 insertions(+), 6 deletions(-)
 create mode 100644 .github/workflows/check-duplicates.yml
 create mode 100644 dev/check-duplicates.mjs

diff --git a/.github/workflows/check-duplicates.yml b/.github/workflows/check-duplicates.yml
new file mode 100644
index 000000000..f2dabf7a6
--- /dev/null
+++ b/.github/workflows/check-duplicates.yml
@@ -0,0 +1,119 @@
+name: Check duplicated content
+
+# Reports passages a pull request adds that say the same thing as a passage
+# elsewhere under docs/, compared with the merge base, as a summary comment
+# plus an inline comment on each added passage. Duplication already on the
+# base branch is ignored. Advisory: never fails the PR.
+
+on:
+    pull_request:
+        paths:
+            - 'docs/**'
+            - 'dev/check-duplicates.mjs'
+            - 'dev/sync-review-comments.sh'
+            - '.github/workflows/check-duplicates.yml'
+
+# A new push supersedes the run for the previous one
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+    cancel-in-progress: true
+
+permissions:
+    contents: read
+    pull-requests: write
+
+jobs:
+    check-duplicates:
+        name: Duplicated content introduced by this PR (advisory)
+        runs-on: ubuntu-latest
+        steps:
+            - name: Check out pull request head
+              uses: actions/checkout@v4
+              with:
+                  ref: ${{ github.event.pull_request.head.sha }}
+                  fetch-depth: 0
+
+            - name: Check out merge base
+              env:
+                  BASE_SHA: ${{ github.event.pull_request.base.sha }}
+              run: |
+                  merge_base=$(git merge-base "$BASE_SHA" HEAD)
+                  git worktree add "$RUNNER_TEMP/base" "$merge_base"
+                  git diff -U0 "$merge_base" HEAD -- docs > "$RUNNER_TEMP/changes.diff"
+
+            # The script exits 1 both with findings and when it crashes. It writes
+            # its output in one go at the end, so a crash leaves it empty. A crash
+            # does not fail the check, since it says nothing about the PR; the
+            # comment step reports it instead of posting the empty report.
+            - name: Record duplicated passages already present on the base branch
+              run: |
+                  node dev/check-duplicates.mjs --format json \
+                      --root "$RUNNER_TEMP/base" > "$RUNNER_TEMP/base-duplicates.json" || true
+
+            - name: Find duplicated passages introduced by this PR
+              id: check
+              env:
+                  # File links in the report open the file on the PR branch
+                  LINK_BASE: ${{ github.event.pull_request.head.repo.html_url }}/blob/${{ github.event.pull_request.head.ref }}
+              run: |
+                  if ! [ -s "$RUNNER_TEMP/base-duplicates.json" ]; then
+                      echo "::warning::check-duplicates crashed on the base branch, so this PR was not checked"
+                      echo "result=crashed" >> "$GITHUB_OUTPUT"
+                      exit 0
+                  fi
+                  if node dev/check-duplicates.mjs --format markdown \
+                      --baseline "$RUNNER_TEMP/base-duplicates.json" \
+                      --diff "$RUNNER_TEMP/changes.diff" \
+                      --review "$RUNNER_TEMP/review.json" \
+                      --link-base "$LINK_BASE" > "$RUNNER_TEMP/report.md"; then
+                      echo "result=clean" >> "$GITHUB_OUTPUT"
+                  elif [ -s "$RUNNER_TEMP/report.md" ]; then
+                      echo "result=found" >> "$GITHUB_OUTPUT"
+                  else
+                      echo "::warning::check-duplicates crashed, so this PR was not checked"
+                      echo "result=crashed" >> "$GITHUB_OUTPUT"
+                  fi
+                  cat "$RUNNER_TEMP/report.md"
+
+            - name: Comment on the pull request
+              # Fork PRs get a read-only token; the report is still in the job log
+              if: github.event.pull_request.head.repo.full_name == github.repository
+              env:
+                  GH_TOKEN: ${{ github.token }}
+                  PR_NUMBER: ${{ github.event.pull_request.number }}
+                  RESULT: ${{ steps.check.outputs.result }}
+                  RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+              run: |
+                  marker='<!-- check-duplicates-report -->'
+                  existing_comment=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \
+                      --paginate --jq ".[] | select(.body | startswith(\"$marker\")) | .id" | head -n 1)
+
+                  # Comment when there is something to report, or an earlier report to resolve
+                  case "$RESULT" in
+                      found)
+                          { echo "$marker"; cat "$RUNNER_TEMP/report.md"; } > "$RUNNER_TEMP/comment.md" ;;
+                      crashed)
+                          printf '%s\n### ⚠️ The duplicated content check could not run on this revision\n\nThis is a problem with the check, not with this PR; see the [job log](%s).\n' \
+                              "$marker" "$RUN_URL" > "$RUNNER_TEMP/comment.md" ;;
+                      *)
+                          [ -n "$existing_comment" ] || exit 0
+                          { echo "$marker"; cat "$RUNNER_TEMP/report.md"; } > "$RUNNER_TEMP/comment.md" ;;
+                  esac
+
+                  if [ -n "$existing_comment" ]; then
+                      gh api --method PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing_comment" \
+                          --field body=@"$RUNNER_TEMP/comment.md"
+                  else
+                      gh pr comment "$PR_NUMBER" --body-file "$RUNNER_TEMP/comment.md"
+                  fi
+
+            - name: Point out the duplicates as review comments
+              # One comment per added passage, kept in sync with the findings;
+              # see dev/sync-review-comments.sh
+              if: >-
+                  github.event.pull_request.head.repo.full_name == github.repository
+                  && contains(fromJSON('["clean", "found"]'), steps.check.outputs.result)
+              env:
+                  GH_TOKEN: ${{ github.token }}
+                  PR_NUMBER: ${{ github.event.pull_request.number }}
+              run: dev/sync-review-comments.sh '<!-- check-duplicates-finding:' "$RUNNER_TEMP/review.json"
diff --git a/AGENTS.md b/AGENTS.md
index fe88818e4..2f95eb1f8 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -64,6 +64,22 @@
   `ai/models.mdx`, `cody/capabilities/supported-models.mdx`,
   `admin/telemetry/`); fix those in sourcegraph/sourcegraph instead.
   `git log --author=sourcegraph-buildkite -- <file>` shows whether a page is one
+- **Duplicated content**: `.github/workflows/check-duplicates.yml` runs
+  `dev/check-duplicates.mjs` and comments the passages a PR adds that say the
+  same thing as a passage elsewhere under `docs/`, paraphrased or verbatim
+  (advisory, never fails the PR). Not part of `pnpm run check`: main has
+  hundreds of pre-existing findings, and CI only reports the ones a PR adds.
+  `node dev/check-duplicates.mjs --diff <(git diff -U0 origin/main)` runs it
+  on your branch; without `--diff` it lists every duplicate in `docs/`
+
+### One home per fact
+
+- Each fact has one home. Link to it instead of copying it, so the copies
+  cannot drift apart.
+- Before writing a config snippet, `rg` the key under `docs/` and link to the
+  existing one.
+- The `SCHEMA_SYNC_START`…`SCHEMA_SYNC_END` blocks are the home for every
+  setting's default. Never restate a default elsewhere.
 
 ### Links
 
@@ -83,12 +99,13 @@ pnpm run check links --check-anchors --check-self-links \
 
 ### PR check comments
 
-The spelling, links, redirects, and example hostnames checks comment on the
-PR: a summary comment per check, plus an inline review comment with a
-`suggestion` block on each flagged line. After pushing, read them and follow
-the instructions they give (fix the spelling, the link, the redirect, or the
-hostname); do not work around a finding, and add a word to
-`cspell-allow-list.txt` only when it is correct.
+The spelling, links, redirects, example hostnames, and duplicated content
+checks comment on the PR: a summary comment per check, plus an inline review
+comment on each flagged line, with a `suggestion` block where the fix is
+mechanical. After pushing, read them and follow the instructions they give
+(fix the spelling, the link, the redirect, or the hostname; replace the
+duplicated passage with a link to its home); do not work around a finding,
+and add a word to `cspell-allow-list.txt` only when it is correct.
 
 ```sh
 gh pr checks <pr>
diff --git a/cspell-allow-list.txt b/cspell-allow-list.txt
index d700988ac..047a304fc 100644
--- a/cspell-allow-list.txt
+++ b/cspell-allow-list.txt
@@ -274,6 +274,7 @@ istari
 itable
 iterm
 itoa
+Jaccard
 JCEF
 jiapantw
 jiti
diff --git a/dev/check-duplicates.mjs b/dev/check-duplicates.mjs
new file mode 100644
index 000000000..a43028359
--- /dev/null
+++ b/dev/check-duplicates.mjs
@@ -0,0 +1,523 @@
+#!/usr/bin/env node
+
+/**
+ * Reports passages under docs/ that say the same thing in more than one place.
+ *
+ * Each fact should have one home, with other pages linking to it: a copy
+ * drifts out of date the moment the original changes. This finds the copies.
+ * Every paragraph, list, table, and fenced code block is normalized
+ * (lowercase; Markdown, JSX, and link targets stripped; words only) and
+ * compared with every other by the Jaccard similarity of their word trigrams,
+ * so paraphrases are caught, not only verbatim copies. Passages shorter than
+ * --min-words are skipped, as are the auto-generated SCHEMA_SYNC blocks: fix
+ * those upstream.
+ *
+ * The pull request workflow (.github/workflows/check-duplicates.yml) runs this
+ * with --baseline and --diff, so a PR hears only about duplication it adds; it
+ * never fails the PR. main has hundreds of pre-existing findings.
+ *
+ * Usage: node dev/check-duplicates.mjs [options]
+ *   --root <dir>         Repository to check (default: this repository)
+ *   --threshold <0-1>    Report pairs at least this similar (default: 0.5)
+ *   --min-words <n>      Skip passages with fewer words (default: 20)
+ *   --format <name>      Output as text (default), json, or markdown
+ *   --baseline <file>    Only report pairs absent from this JSON file (produced
+ *                        by --format json on another revision). Pairs are
+ *                        matched by the file and heading of each passage, so
+ *                        editing a pre-existing duplicate does not report it
+ *   --diff <file>        Unified diff, e.g. from `git diff -U0 origin/main`;
+ *                        only report pairs where a passage has an added line
+ *   --link-base <url>    Markdown output links each passage to <url>/<path>,
+ *                        e.g. https://github.com/sourcegraph/docs/blob/<branch>
+ *   --review <file>      Write a GitHub pull request review (JSON body for
+ *                        POST /repos/{owner}/{repo}/pulls/{n}/reviews) with one
+ *                        comment per added passage that duplicates another.
+ *                        Requires --diff
+ *
+ * Exits 1 when any finding is reported.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import {fileURLToPath} from 'url';
+
+const args = process.argv.slice(2);
+const ROOT_DIR = path.resolve(
+	flagValue('--root') ??
+		path.dirname(path.dirname(fileURLToPath(import.meta.url)))
+);
+const DOCS_DIR = path.join(ROOT_DIR, 'docs');
+const THRESHOLD = Number(flagValue('--threshold') ?? 0.5);
+const MIN_WORDS = Number(flagValue('--min-words') ?? 20);
+const FORMAT = flagValue('--format') ?? 'text';
+const BASELINE_FILE = flagValue('--baseline');
+const DIFF = parseDiff(flagValue('--diff'));
+const LINK_BASE = flagValue('--link-base')?.replace(/\/$/, '');
+const REVIEW_FILE = flagValue('--review');
+
+// Pages the `🤖 Sync generated docs` PRs overwrite whole (see AGENTS.md); their
+// repetition is the generator's, and is fixed in sourcegraph/sourcegraph
+const GENERATED_PAGES = [
+	'docs/admin/telemetry/',
+	'docs/ai/models.mdx',
+	'docs/cli/references/',
+	'docs/cody/capabilities/supported-models.mdx',
+	'docs/self-hosted/observability/alerts.mdx',
+	'docs/self-hosted/observability/dashboards.mdx'
+];
+
+const SHINGLE_SIZE = 3;
+// A trigram in more passages than this is boilerplate ("in the site
+// configuration"), so it does not nominate candidate pairs
+const MAX_POSTINGS = 50;
+const SNIPPET_LENGTH = 90;
+
+if (REVIEW_FILE && !DIFF) {
+	throw new Error('--review needs --diff, to know which lines were added');
+}
+
+function flagValue(name) {
+	const index = args.indexOf(name);
+	return index === -1 ? undefined : args[index + 1];
+}
+
+// Added lines of a unified diff, by path relative to the repository:
+// Map<'docs/foo.mdx', Set<lineNumber>>
+function parseDiff(file) {
+	if (!file) return undefined;
+	const addedLines = new Map();
+	let currentFile;
+	let lineNumber;
+	for (const line of fs.readFileSync(file, 'utf-8').split('\n')) {
+		if (line.startsWith('diff --git ')) {
+			currentFile = undefined;
+		} else if (line.startsWith('+++ ') && !currentFile) {
+			// `+++ /dev/null` is a deleted file, which has no added lines
+			currentFile = line.slice(4).replace(/^b\//, '');
+			if (currentFile === '/dev/null') continue;
+			addedLines.set(currentFile, new Set());
+		} else if (line.startsWith('@@ ')) {
+			lineNumber = Number(line.match(/^@@ -\S+ \+(\d+)/)[1]);
+		} else if (line.startsWith('+') && currentFile) {
+			addedLines.get(currentFile).add(lineNumber++);
+		} else if (line.startsWith(' ')) {
+			lineNumber++;
+		}
+	}
+	return addedLines;
+}
+
+// Sorted relative paths of the .md and .mdx files under docs/ that are
+// written by hand
+function listDocsFiles(dir = DOCS_DIR) {
+	return fs
+		.readdirSync(dir, {withFileTypes: true})
+		.sort((a, b) => a.name.localeCompare(b.name))
+		.flatMap(entry => {
+			const full = path.join(dir, entry.name);
+			if (entry.isDirectory()) return listDocsFiles(full);
+			const file = path.relative(ROOT_DIR, full);
+			return /\.mdx?$/.test(entry.name) &&
+				!GENERATED_PAGES.some(prefix => file.startsWith(prefix))
+				? [file]
+				: [];
+		});
+}
+
+// The words of a passage, lowercased, without Markdown, JSX, MDX comments,
+// link targets, or URLs, so the same sentence formatted two ways compares equal
+function words(text) {
+	return (
+		text
+			.toLowerCase()
+			.replace(/\{\/\*[\s\S]*?\*\/\}/g, ' ')
+			.replace(/<[^>]*>/g, ' ')
+			.replace(/\]\([^)]*\)/g, ']')
+			.replace(/https?:\/\/\S+/g, ' ')
+			.match(/[a-z0-9]+/g) ?? []
+	);
+}
+
+function shingles(wordList) {
+	const set = new Set();
+	for (let index = 0; index + SHINGLE_SIZE <= wordList.length; index++) {
+		set.add(wordList.slice(index, index + SHINGLE_SIZE).join(' '));
+	}
+	return set;
+}
+
+// The start of a passage for the report: without code fences, MDX comments,
+// or JSX tags, since a snippet of `<Callout type="warning">` identifies
+// nothing, and without backticks, since a cut-off one would swallow the rest
+// of a Markdown line
+function snippet(text) {
+	const content = text
+		.replace(/^\s*(`{3,}|~{3,}).*$/gm, ' ')
+		.replace(/\{\/\*[\s\S]*?\*\/\}/g, ' ')
+		.replace(/<[^>]*>/g, ' ')
+		.replace(/`/g, '')
+		.replace(/\s+/g, ' ')
+		.trim();
+	return content.length > SNIPPET_LENGTH
+		? content.slice(0, SNIPPET_LENGTH - 1) + '…'
+		: content;
+}
+
+// The passages of one file that are long enough to compare:
+// [{file, line, endLine, heading, snippet, shingles}]. A passage is a run of
+// non-blank lines (paragraph, list, or table) or a fenced code block. Front
+// matter, headings, imports, and SCHEMA_SYNC blocks are not passages; the
+// heading is kept to identify the passage across revisions.
+function extractPassages(file, source) {
+	const lines = source.split('\n');
+	const passages = [];
+	let heading = '';
+	let inSchemaSync = false;
+	let fence; // the ``` or ~~~ that opened the code block being collected
+	let start; // index of the first line being collected
+	let collected = [];
+
+	const flush = () => {
+		const text = collected.join('\n');
+		const wordList = words(text);
+		if (
+			wordList.length >= MIN_WORDS &&
+			!/^\s*(import|export)\s/.test(text)
+		) {
+			passages.push({
+				file,
+				line: start + 1,
+				endLine: start + collected.length,
+				heading,
+				snippet: snippet(text),
+				shingles: shingles(wordList)
+			});
+		}
+		collected = [];
+	};
+
+	let index = lines[0] === '---' ? lines.indexOf('---', 1) + 1 : 0;
+	for (; index < lines.length; index++) {
+		const text = lines[index];
+		if (fence) {
+			collected.push(text);
+			if (text.trim().startsWith(fence)) {
+				fence = undefined;
+				flush();
+			}
+			continue;
+		}
+		if (text.includes('SCHEMA_SYNC_START')) inSchemaSync = true;
+		if (text.includes('SCHEMA_SYNC_END')) inSchemaSync = false;
+		if (inSchemaSync) continue;
+
+		const fenceMatch = text.match(/^\s*(`{3,}|~{3,})/);
+		const headingMatch = text.match(/^#{1,6}\s+(.*)/);
+		// A list item is its own passage, so one step copied between two
+		// procedures is found; its indented continuation lines stay with it
+		const listItem = /^\s*(?:[-*+]|\d+\.)\s/.test(text);
+		if (fenceMatch || headingMatch || listItem || text.trim() === '')
+			flush();
+		if (fenceMatch) {
+			fence = fenceMatch[1];
+			start = index;
+			collected.push(text);
+		} else if (headingMatch) {
+			heading = headingMatch[1].trim();
+		} else if (text.trim() !== '') {
+			if (collected.length === 0) start = index;
+			collected.push(text);
+		}
+	}
+	flush();
+	return passages;
+}
+
+function jaccard(a, b) {
+	const [small, big] = a.size <= b.size ? [a, b] : [b, a];
+	let intersection = 0;
+	for (const shingle of small) {
+		if (big.has(shingle)) intersection++;
+	}
+	return intersection / (a.size + b.size - intersection);
+}
+
+// Every pair of passages at least THRESHOLD similar: [{similarity, a, b}],
+// with a before b in file order. Candidates come from an inverted index of
+// trigrams, so only passages that share an uncommon trigram are compared.
+function findDuplicatePairs(passages) {
+	const index = new Map();
+	passages.forEach((passage, id) => {
+		for (const shingle of passage.shingles) {
+			if (!index.has(shingle)) index.set(shingle, []);
+			index.get(shingle).push(id);
+		}
+	});
+
+	const pairs = [];
+	passages.forEach((a, aId) => {
+		const candidates = new Set();
+		for (const shingle of a.shingles) {
+			const ids = index.get(shingle);
+			if (ids.length > MAX_POSTINGS) continue;
+			for (const bId of ids) {
+				if (bId > aId) candidates.add(bId);
+			}
+		}
+		for (const bId of candidates) {
+			const b = passages[bId];
+			// Jaccard cannot exceed the size ratio, so skip hopeless pairs
+			const [small, big] =
+				a.shingles.size <= b.shingles.size
+					? [a.shingles.size, b.shingles.size]
+					: [b.shingles.size, a.shingles.size];
+			if (small / big < THRESHOLD) continue;
+			const similarity = jaccard(a.shingles, b.shingles);
+			if (similarity >= THRESHOLD) pairs.push({similarity, a, b});
+		}
+	});
+	return pairs;
+}
+
+// The passage without its shingles, for the JSON output and the baseline
+function describe({shingles: _, ...passage}) {
+	return passage;
+}
+
+// Line numbers are left out so a passage that only moved is not a new finding
+function passageKey({file, heading}) {
+	return `${file}#${heading}`;
+}
+
+function pairKey({a, b}) {
+	return [passageKey(a), passageKey(b)].sort().join('\u0000');
+}
+
+function withoutBaseline(pairs, baselineFile) {
+	const baseline = new Set(
+		JSON.parse(fs.readFileSync(baselineFile, 'utf-8')).map(pairKey)
+	);
+	return pairs.filter(pair => !baseline.has(pairKey(pair)));
+}
+
+// The first line of the passage that the diff added, or undefined
+function addedLineIn({file, line, endLine}) {
+	const added = DIFF?.get(file);
+	if (!added) return undefined;
+	for (let current = line; current <= endLine; current++) {
+		if (added.has(current)) return current;
+	}
+	return undefined;
+}
+
+// Passages joined by a reported pair, grouped: [{passages, similarity}] with
+// passages in file order and `similarity` the weakest link in the group,
+// biggest groups first
+function clusters(pairs) {
+	const parent = new Map();
+	const find = passage => {
+		while (parent.has(passage) && parent.get(passage) !== passage) {
+			passage = parent.get(passage);
+		}
+		return passage;
+	};
+	for (const {a, b} of pairs) {
+		parent.set(find(a), find(b));
+	}
+
+	const groups = new Map();
+	for (const pair of pairs) {
+		const root = find(pair.a);
+		if (!groups.has(root)) {
+			groups.set(root, {passages: new Set(), similarity: 1});
+		}
+		const group = groups.get(root);
+		group.passages.add(pair.a).add(pair.b);
+		group.similarity = Math.min(group.similarity, pair.similarity);
+	}
+
+	return [...groups.values()]
+		.map(({passages, similarity}) => ({
+			similarity,
+			passages: [...passages].sort(
+				(a, b) => a.file.localeCompare(b.file) || a.line - b.line
+			)
+		}))
+		.sort(
+			(a, b) =>
+				b.passages.length - a.passages.length ||
+				b.similarity - a.similarity
+		);
+}
+
+function percent(similarity) {
+	return `${Math.round(similarity * 100)}%`;
+}
+
+function location({file, line, endLine}) {
+	return `${file}:${line}${endLine > line ? `-${endLine}` : ''}`;
+}
+
+function formatText(pairs) {
+	const scope = BASELINE_FILE || DIFF ? 'new ' : '';
+	if (pairs.length === 0) {
+		return `✅ No ${scope}duplicated passages found!\n`;
+	}
+	const groups = clusters(pairs);
+	const lines = [
+		`❌ Found ${pairs.length} ${scope}duplicated pair(s) of passages in ${groups.length} group(s):`
+	];
+	for (const {passages, similarity} of groups) {
+		lines.push(
+			'',
+			`${passages.length} passages at least ${percent(similarity)} similar:`
+		);
+		for (const passage of passages) {
+			lines.push(`   ${location(passage)}`, `   └─ ${passage.snippet}`);
+		}
+	}
+	return lines.join('\n') + '\n';
+}
+
+function linkTo(text, url) {
+	return url ? `[${text}](${url})` : text;
+}
+
+// ?plain=1 opens GitHub's code view, where #L<n> anchors work; the rendered
+// Markdown preview ignores them
+function passageLink(passage) {
+	const {file, line, endLine} = passage;
+	const anchor = endLine > line ? `#L${line}-L${endLine}` : `#L${line}`;
+	return linkTo(
+		`\`${location(passage)}\``,
+		LINK_BASE && `${LINK_BASE}/${file}?plain=1${anchor}`
+	);
+}
+
+const CONVENTION =
+	'Each fact has one home in the docs. Keep the passage on the page that owns ' +
+	'the topic, and replace the others with a sentence and a link to it, so they ' +
+	'cannot drift apart. Before writing a config snippet, search `docs/` for the ' +
+	'key and link to the existing one.';
+
+// Body for a pull request comment
+function formatMarkdown(pairs) {
+	if (pairs.length === 0) {
+		return '### ✅ This revision adds no duplicated passages\n';
+	}
+	const groups = clusters(pairs);
+	const lines = [
+		`### ⚠️ This PR adds ${pairs.length} duplicated pair(s) of passages`,
+		'',
+		'Only passages this PR added are listed, with the places that already say ' +
+			'the same thing; duplication already on the base branch is not. ' +
+			'Each added passage also has an inline comment.',
+		''
+	];
+	for (const {passages, similarity} of groups) {
+		lines.push(
+			`**${passages.length} passages, at least ${percent(similarity)} similar**`,
+			...passages.map(
+				passage => `- ${passageLink(passage)}: ${passage.snippet}`
+			),
+			''
+		);
+	}
+	lines.push(
+		CONVENTION,
+		'',
+		'Reproduce locally with `node dev/check-duplicates.mjs --diff <(git diff -U0 origin/main)`.'
+	);
+	return lines.join('\n') + '\n';
+}
+
+// First line of a review comment, so the workflow can match the comments it
+// posted earlier to the findings still present and delete the rest
+const REVIEW_MARKER = '<!-- check-duplicates-finding:';
+
+// Body for POST /repos/{owner}/{repo}/pulls/{n}/reviews: one comment per added
+// passage, on its first added line, naming the passages it duplicates. No
+// review body: a submitted review cannot be deleted, so a body would outlive
+// the comments once the duplication is gone.
+function reviewRequest(pairs) {
+	const partners = new Map(); // added passage → [{passage, similarity}]
+	for (const {similarity, a, b} of pairs) {
+		for (const [passage, partner] of [
+			[a, b],
+			[b, a]
+		]) {
+			if (addedLineIn(passage) === undefined) continue;
+			if (!partners.has(passage)) partners.set(passage, []);
+			partners.get(passage).push({passage: partner, similarity});
+		}
+	}
+	const comments = [...partners].map(([passage, duplicates]) => ({
+		path: passage.file,
+		line: addedLineIn(passage),
+		side: 'RIGHT',
+		body: [
+			`${REVIEW_MARKER} ${duplicates.map(({passage}) => passageKey(passage)).join(', ')} -->`,
+			'This passage says the same thing as:',
+			...duplicates.map(
+				({passage, similarity}) =>
+					`- ${passageLink(passage)} (${percent(similarity)} similar)`
+			),
+			'',
+			CONVENTION
+		].join('\n')
+	}));
+	return {event: 'COMMENT', body: '', comments};
+}
+
+const FORMATTERS = {
+	text: formatText,
+	json: pairs =>
+		JSON.stringify(
+			pairs.map(({similarity, a, b}) => ({
+				similarity,
+				a: describe(a),
+				b: describe(b)
+			})),
+			null,
+			'\t'
+		) + '\n',
+	markdown: formatMarkdown
+};
+
+function main() {
+	const format = FORMATTERS[FORMAT];
+	if (!format) {
+		throw new Error(
+			`Unknown --format "${FORMAT}"; use text, json, or markdown`
+		);
+	}
+
+	const passages = listDocsFiles().flatMap(file =>
+		extractPassages(
+			file,
+			fs.readFileSync(path.join(ROOT_DIR, file), 'utf-8')
+		)
+	);
+	let pairs = findDuplicatePairs(passages);
+	if (BASELINE_FILE) {
+		pairs = withoutBaseline(pairs, BASELINE_FILE);
+	}
+	if (DIFF) {
+		pairs = pairs.filter(
+			({a, b}) =>
+				addedLineIn(a) !== undefined || addedLineIn(b) !== undefined
+		);
+	}
+	if (REVIEW_FILE) {
+		fs.writeFileSync(
+			REVIEW_FILE,
+			JSON.stringify(reviewRequest(pairs), null, '\t') + '\n'
+		);
+	}
+
+	// Not process.exit(): that can truncate stdout when it is a pipe
+	process.stdout.write(format(pairs));
+	process.exitCode = pairs.length === 0 ? 0 : 1;
+}
+
+main();

From f98c8578b523369c5d2b1156e42d14305945802b Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 6 Oct 2026 16:36:09 -0600
Subject: [PATCH 2/4] check-duplicates: report edited passages that were
 already duplicated

A PR that edits one copy of a duplicated passage now gets a comment
naming the other copies, so the author can pick the home and replace
the rest with links, or make the same change there. Added duplication
and edited duplication are reported in separate sections with their
own advice. Also strip trailing spaces from the example-hostnames
bullet in AGENTS.md.

Amp-Thread-ID: https://ampcode.com/threads/T-01a11326-478d-72cf-bbb7-587d16b378e9
Co-authored-by: Amp <amp@ampcode.com>
---
 AGENTS.md                |  36 ++++----
 dev/check-duplicates.mjs | 186 ++++++++++++++++++++++++++-------------
 2 files changed, 142 insertions(+), 80 deletions(-)

diff --git a/AGENTS.md b/AGENTS.md
index 2f95eb1f8..13fd6e54d 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -40,23 +40,23 @@
   Add product names and identifiers to `cspell-allow-list.txt`, in
   alphabetical order; the check reports out-of-order entries. CSpell is not a
   project dependency: `npx cspell@10 --no-progress <file>` to run it locally
-- **Example hostnames**: placeholder hostnames in docs are `*.example.com` 
-  (`sourcegraph.example.com` for the reader's instance, 
-  `github.example.com` for a code host, 
-  `https://sourcegraph.example.com` where a whole URL is meant); 
+- **Example hostnames**: placeholder hostnames in docs are `*.example.com`
+  (`sourcegraph.example.com` for the reader's instance,
+  `github.example.com` for a code host,
+  `https://sourcegraph.example.com` where a whole URL is meant);
   never `<URL>`, `[hostname]`, or a `$VARIABLE` the reader has to guess.
-  Example URLs use `https://`, except where needed, eg. loopback, 
-  in-cluster, proxy, and XML-namespace addresses. 
-  In a shell snippet a `$VARIABLE` is fine only when the same snippet 
-  exports it with an example value. 
+  Example URLs use `https://`, except where needed, eg. loopback,
+  in-cluster, proxy, and XML-namespace addresses.
+  In a shell snippet a `$VARIABLE` is fine only when the same snippet
+  exports it with an example value.
   `.github/workflows/check-hostnames.yml` runs `dev/check-hostnames.mjs`
   on the lines a PR adds and comments the findings with suggested changes
-  (advisory, never fails the PR). 
-  `dev/example-hostnames.json` maps each recommended hostname to the 
-  placeholders seen in its place, 
-  as either literal text,  a `*` glob, or a `/regex/`, 
-  matched case-insensitively with `-`, `_`, and `.` interchangeable; 
-  add to it when the check misses one, keeping keys and lists sorted. 
+  (advisory, never fails the PR).
+  `dev/example-hostnames.json` maps each recommended hostname to the
+  placeholders seen in its place,
+  as either literal text,  a `*` glob, or a `/regex/`,
+  matched case-insensitively with `-`, `_`, and `.` interchangeable;
+  add to it when the check misses one, keeping keys and lists sorted.
   `node dev/check-hostnames.mjs` checks all of `docs/`
   The `🤖 Sync generated docs` PRs overwrite the
   `SCHEMA_SYNC_START`…`SCHEMA_SYNC_END` blocks and whole pages
@@ -66,9 +66,11 @@
   `git log --author=sourcegraph-buildkite -- <file>` shows whether a page is one
 - **Duplicated content**: `.github/workflows/check-duplicates.yml` runs
   `dev/check-duplicates.mjs` and comments the passages a PR adds that say the
-  same thing as a passage elsewhere under `docs/`, paraphrased or verbatim
-  (advisory, never fails the PR). Not part of `pnpm run check`: main has
-  hundreds of pre-existing findings, and CI only reports the ones a PR adds.
+  same thing as a passage elsewhere under `docs/`, paraphrased or verbatim,
+  and the passages a PR edits that were already duplicated, so the author can
+  pick the home and replace the other copy with a link (advisory, never fails
+  the PR). Not part of `pnpm run check`: main has hundreds of pre-existing
+  findings, and CI only reports the ones a PR adds or edits.
   `node dev/check-duplicates.mjs --diff <(git diff -U0 origin/main)` runs it
   on your branch; without `--diff` it lists every duplicate in `docs/`
 
diff --git a/dev/check-duplicates.mjs b/dev/check-duplicates.mjs
index a43028359..e6d56e714 100644
--- a/dev/check-duplicates.mjs
+++ b/dev/check-duplicates.mjs
@@ -13,7 +13,8 @@
  * those upstream.
  *
  * The pull request workflow (.github/workflows/check-duplicates.yml) runs this
- * with --baseline and --diff, so a PR hears only about duplication it adds; it
+ * with --baseline and --diff, so a PR hears about the duplication it adds, and
+ * about the other copies of a passage it edits, which now disagree with it; it
  * never fails the PR. main has hundreds of pre-existing findings.
  *
  * Usage: node dev/check-duplicates.mjs [options]
@@ -21,10 +22,10 @@
  *   --threshold <0-1>    Report pairs at least this similar (default: 0.5)
  *   --min-words <n>      Skip passages with fewer words (default: 20)
  *   --format <name>      Output as text (default), json, or markdown
- *   --baseline <file>    Only report pairs absent from this JSON file (produced
- *                        by --format json on another revision). Pairs are
- *                        matched by the file and heading of each passage, so
- *                        editing a pre-existing duplicate does not report it
+ *   --baseline <file>    JSON from --format json on another revision. Pairs it
+ *                        already has, matched by the file and heading of each
+ *                        passage, are reported apart as edited rather than
+ *                        added, and only when --diff touches them
  *   --diff <file>        Unified diff, e.g. from `git diff -U0 origin/main`;
  *                        only report pairs where a passage has an added line
  *   --link-base <url>    Markdown output links each passage to <url>/<path>,
@@ -293,11 +294,15 @@ function pairKey({a, b}) {
 	return [passageKey(a), passageKey(b)].sort().join('\u0000');
 }
 
-function withoutBaseline(pairs, baselineFile) {
+// The pairs with `preexisting` set where the baseline already had the pair
+function markPreexisting(pairs, baselineFile) {
 	const baseline = new Set(
 		JSON.parse(fs.readFileSync(baselineFile, 'utf-8')).map(pairKey)
 	);
-	return pairs.filter(pair => !baseline.has(pairKey(pair)));
+	return pairs.map(pair => ({
+		...pair,
+		preexisting: baseline.has(pairKey(pair))
+	}));
 }
 
 // The first line of the passage that the diff added, or undefined
@@ -358,16 +363,19 @@ function location({file, line, endLine}) {
 	return `${file}:${line}${endLine > line ? `-${endLine}` : ''}`;
 }
 
-function formatText(pairs) {
-	const scope = BASELINE_FILE || DIFF ? 'new ' : '';
-	if (pairs.length === 0) {
-		return `✅ No ${scope}duplicated passages found!\n`;
-	}
-	const groups = clusters(pairs);
-	const lines = [
-		`❌ Found ${pairs.length} ${scope}duplicated pair(s) of passages in ${groups.length} group(s):`
-	];
-	for (const {passages, similarity} of groups) {
+// With --baseline, the pairs the change added and the ones it only edited:
+// an author touching one copy should hear about the other, since the two
+// now disagree, but the advice differs. Without one, every pair is "added".
+function splitByBaseline(pairs) {
+	return {
+		added: pairs.filter(pair => !pair.preexisting),
+		edited: pairs.filter(pair => pair.preexisting)
+	};
+}
+
+function textGroups(pairs) {
+	const lines = [];
+	for (const {passages, similarity} of clusters(pairs)) {
 		lines.push(
 			'',
 			`${passages.length} passages at least ${percent(similarity)} similar:`
@@ -376,6 +384,29 @@ function formatText(pairs) {
 			lines.push(`   ${location(passage)}`, `   └─ ${passage.snippet}`);
 		}
 	}
+	return lines;
+}
+
+function formatText(pairs) {
+	const scope = DIFF ? ' in this change' : '';
+	if (pairs.length === 0) {
+		return `✅ No duplicated passages found${scope}!\n`;
+	}
+	const {added, edited} = splitByBaseline(pairs);
+	const lines = [];
+	if (added.length > 0) {
+		lines.push(
+			`❌ Found ${added.length} duplicated pair(s) of passages${scope} in ${clusters(added).length} group(s):`,
+			...textGroups(added)
+		);
+	}
+	if (edited.length > 0) {
+		lines.push(
+			...(lines.length > 0 ? [''] : []),
+			`⚠️ ${edited.length} edited passage pair(s) were already duplicated in ${clusters(edited).length} group(s):`,
+			...textGroups(edited)
+		);
+	}
 	return lines.join('\n') + '\n';
 }
 
@@ -400,32 +431,50 @@ const CONVENTION =
 	'cannot drift apart. Before writing a config snippet, search `docs/` for the ' +
 	'key and link to the existing one.';
 
+const EDITED_ADVICE =
+	'The passage you changed is also written somewhere else, so the two copies ' +
+	'now disagree. Decide which page is the home for it, and replace the other ' +
+	'copy with a link; or make the same change there too.';
+
+function markdownGroups(pairs) {
+	return clusters(pairs).flatMap(({passages, similarity}) => [
+		`**${passages.length} passages, at least ${percent(similarity)} similar**`,
+		...passages.map(
+			passage => `- ${passageLink(passage)}: ${passage.snippet}`
+		),
+		''
+	]);
+}
+
 // Body for a pull request comment
 function formatMarkdown(pairs) {
 	if (pairs.length === 0) {
-		return '### ✅ This revision adds no duplicated passages\n';
+		return '### ✅ This revision adds no duplicated passages, and edits none\n';
 	}
-	const groups = clusters(pairs);
-	const lines = [
-		`### ⚠️ This PR adds ${pairs.length} duplicated pair(s) of passages`,
-		'',
-		'Only passages this PR added are listed, with the places that already say ' +
-			'the same thing; duplication already on the base branch is not. ' +
-			'Each added passage also has an inline comment.',
-		''
-	];
-	for (const {passages, similarity} of groups) {
+	const {added, edited} = splitByBaseline(pairs);
+	const lines = [];
+	if (added.length > 0) {
 		lines.push(
-			`**${passages.length} passages, at least ${percent(similarity)} similar**`,
-			...passages.map(
-				passage => `- ${passageLink(passage)}: ${passage.snippet}`
-			),
+			`### ⚠️ This PR adds ${added.length} duplicated pair(s) of passages`,
+			'',
+			'Passages this PR added, with the places that already say the same ' +
+				'thing. Each added passage also has an inline comment.',
+			'',
+			...markdownGroups(added),
+			CONVENTION,
 			''
 		);
 	}
+	if (edited.length > 0) {
+		lines.push(
+			`### ⚠️ This PR edits ${edited.length} passage pair(s) that were already duplicated`,
+			'',
+			EDITED_ADVICE,
+			'',
+			...markdownGroups(edited)
+		);
+	}
 	lines.push(
-		CONVENTION,
-		'',
 		'Reproduce locally with `node dev/check-duplicates.mjs --diff <(git diff -U0 origin/main)`.'
 	);
 	return lines.join('\n') + '\n';
@@ -435,37 +484,45 @@ function formatMarkdown(pairs) {
 // posted earlier to the findings still present and delete the rest
 const REVIEW_MARKER = '<!-- check-duplicates-finding:';
 
-// Body for POST /repos/{owner}/{repo}/pulls/{n}/reviews: one comment per added
-// passage, on its first added line, naming the passages it duplicates. No
-// review body: a submitted review cannot be deleted, so a body would outlive
-// the comments once the duplication is gone.
+// Body for POST /repos/{owner}/{repo}/pulls/{n}/reviews: one comment per
+// passage the diff touched, on its first added line, naming the passages it
+// duplicates; the advice depends on whether the duplication is new. No review
+// body: a submitted review cannot be deleted, so a body would outlive the
+// comments once the duplication is gone.
 function reviewRequest(pairs) {
-	const partners = new Map(); // added passage → [{passage, similarity}]
-	for (const {similarity, a, b} of pairs) {
+	const partners = new Map(); // touched passage → [{passage, similarity, preexisting}]
+	for (const {similarity, a, b, preexisting} of pairs) {
 		for (const [passage, partner] of [
 			[a, b],
 			[b, a]
 		]) {
 			if (addedLineIn(passage) === undefined) continue;
 			if (!partners.has(passage)) partners.set(passage, []);
-			partners.get(passage).push({passage: partner, similarity});
+			partners
+				.get(passage)
+				.push({passage: partner, similarity, preexisting});
 		}
 	}
-	const comments = [...partners].map(([passage, duplicates]) => ({
-		path: passage.file,
-		line: addedLineIn(passage),
-		side: 'RIGHT',
-		body: [
-			`${REVIEW_MARKER} ${duplicates.map(({passage}) => passageKey(passage)).join(', ')} -->`,
-			'This passage says the same thing as:',
-			...duplicates.map(
-				({passage, similarity}) =>
-					`- ${passageLink(passage)} (${percent(similarity)} similar)`
-			),
-			'',
-			CONVENTION
-		].join('\n')
-	}));
+	const comments = [...partners].map(([passage, duplicates]) => {
+		const onlyEdited = duplicates.every(({preexisting}) => preexisting);
+		return {
+			path: passage.file,
+			line: addedLineIn(passage),
+			side: 'RIGHT',
+			body: [
+				`${REVIEW_MARKER} ${duplicates.map(({passage}) => passageKey(passage)).join(', ')} -->`,
+				onlyEdited
+					? 'This passage was already duplicated, and is now out of step with:'
+					: 'This passage says the same thing as:',
+				...duplicates.map(
+					({passage, similarity}) =>
+						`- ${passageLink(passage)} (${percent(similarity)} similar)`
+				),
+				'',
+				onlyEdited ? EDITED_ADVICE : CONVENTION
+			].join('\n')
+		};
+	});
 	return {event: 'COMMENT', body: '', comments};
 }
 
@@ -500,14 +557,17 @@ function main() {
 	);
 	let pairs = findDuplicatePairs(passages);
 	if (BASELINE_FILE) {
-		pairs = withoutBaseline(pairs, BASELINE_FILE);
-	}
-	if (DIFF) {
-		pairs = pairs.filter(
-			({a, b}) =>
-				addedLineIn(a) !== undefined || addedLineIn(b) !== undefined
-		);
+		pairs = markPreexisting(pairs, BASELINE_FILE);
 	}
+	// With a diff, a pre-existing pair the change edited is still reported, so
+	// the author hears about the other copy; without one, nothing says which
+	// pre-existing pairs matter, so they are all left out
+	pairs = DIFF
+		? pairs.filter(
+				({a, b}) =>
+					addedLineIn(a) !== undefined || addedLineIn(b) !== undefined
+			)
+		: pairs.filter(pair => !pair.preexisting);
 	if (REVIEW_FILE) {
 		fs.writeFileSync(
 			REVIEW_FILE,

From 6ecae68e9c4283c9212612694e15b8f9214ae995 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 6 Oct 2026 17:08:01 -0600
Subject: [PATCH 3/4] vercel: skip the build for CI-only check scripts

dev/check-duplicates.mjs, check-hostnames.mjs, check-redirects.mjs, and
example-hostnames.json only run in PR checks; the site build never reads
them, so a change to them should not deploy a preview.

Amp-Thread-ID: https://ampcode.com/threads/T-01a11326-478d-72cf-bbb7-587d16b378e9
Co-authored-by: Amp <amp@ampcode.com>
---
 dev/vercel-ignore-build.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/dev/vercel-ignore-build.sh b/dev/vercel-ignore-build.sh
index 3f87eda0b..1fe50f23e 100755
--- a/dev/vercel-ignore-build.sh
+++ b/dev/vercel-ignore-build.sh
@@ -13,7 +13,11 @@ exec git diff --quiet HEAD^ HEAD -- . \
 	':(exclude).gitignore' \
 	':(exclude)renovate.json' \
 	':(exclude)cspell*' \
+	':(exclude)dev/check-duplicates.mjs' \
+	':(exclude)dev/check-hostnames.mjs' \
+	':(exclude)dev/check-redirects.mjs' \
 	':(exclude)dev/check-spelling.mjs' \
+	':(exclude)dev/example-hostnames.json' \
 	':(exclude)dev/post-spelling-review.mjs' \
 	':(exclude)dev/report-vercel-build.mjs' \
 	':(exclude)dev/verify-links-live.mjs' \

From 28453f3ec71124f99578e82e3790a4dd94d7503f Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 6 Oct 2026 17:09:47 -0600
Subject: [PATCH 4/4] vercel: the ignore script does not need a build either

Amp-Thread-ID: https://ampcode.com/threads/T-01a11326-478d-72cf-bbb7-587d16b378e9
Co-authored-by: Amp <amp@ampcode.com>
---
 dev/vercel-ignore-build.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dev/vercel-ignore-build.sh b/dev/vercel-ignore-build.sh
index 1fe50f23e..3a1f12896 100755
--- a/dev/vercel-ignore-build.sh
+++ b/dev/vercel-ignore-build.sh
@@ -22,4 +22,5 @@ exec git diff --quiet HEAD^ HEAD -- . \
 	':(exclude)dev/report-vercel-build.mjs' \
 	':(exclude)dev/verify-links-live.mjs' \
 	':(exclude)dev/sync-review-comments.sh' \
+	':(exclude)dev/vercel-ignore-build.sh' \
 	':(exclude)dev/slack-app-vercel-build-report.json'