From 4842fa300313973faaa61fee0b7946b8cd482caa Mon Sep 17 00:00:00 2001 From: bhushan4work Date: Wed, 4 Nov 2026 18:28:28 +0530 Subject: [PATCH 1/3] feat(ingest): crawl discovery source (#586) --- .changeset/crawl-discovery.md | 6 + api/ingest.api.md | 8 + packages/cli/src/main.ts | 14 +- packages/cli/tests/cli.test.ts | 18 ++ packages/ingest/src/discover.ts | 191 ++++++++++++++++++ packages/ingest/src/ingest.ts | 27 ++- packages/ingest/src/types.ts | 6 +- packages/ingest/tests/ingest.test.ts | 138 +++++++++++++ .../src/content/docs/core/knowledge-base.mdx | 1 + 9 files changed, 399 insertions(+), 10 deletions(-) create mode 100644 .changeset/crawl-discovery.md diff --git a/.changeset/crawl-discovery.md b/.changeset/crawl-discovery.md new file mode 100644 index 00000000..7ab425e1 --- /dev/null +++ b/.changeset/crawl-discovery.md @@ -0,0 +1,6 @@ +--- +'@chatcops/ingest': minor +'@chatcops/cli': minor +--- + +Support crawl ingestion sources starting from a URL and following same-origin links breadth-first with maxDepth, maxPages, deduplication, robots.txt compliance, and cached HTML reuse during extraction. Add --crawl to the chatcops ingest command and crawl: URLs to --source. diff --git a/api/ingest.api.md b/api/ingest.api.md index 6cefccb3..98afb906 100644 --- a/api/ingest.api.md +++ b/api/ingest.api.md @@ -27,6 +27,7 @@ export declare function discover(sources: IngestSource[], options: DiscoverOptio export interface DiscoveredPage { url: string; file?: string; + html?: string; } export interface DiscoverOptions { @@ -35,6 +36,8 @@ export interface DiscoverOptions { timeoutMs: number; include?: string[]; exclude?: string[]; + concurrency?: number; + respectRobots?: boolean; } export interface Discovery { @@ -114,6 +117,11 @@ export type IngestSource = { baseUrl: string; path: string; type: 'directory'; +} | { + maxDepth?: number; + maxPages?: number; + type: 'crawl'; + url: string; }; export interface IngestStats { diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index dd7a7f36..5b03c854 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -34,8 +34,9 @@ Usage: chatcops test [files...] [options] Run eval files against your bot Ingest options: - --source A sitemap (.xml, .xml.gz), a page URL, or a folder of .md/.mdx/.html - files (with --base-url). Repeatable. Replaces the config's sources. + --source A sitemap (.xml, .xml.gz), crawl:, a page URL, or a folder of + .md/.mdx/.html files (with --base-url). Repeatable. Replaces the config's sources. + --crawl Crawl same-origin pages starting at url. Replaces the config's sources. --base-url Where a folder source is published --index Index name (default "default") --dry-run Discover, extract and chunk; write and embed nothing @@ -125,6 +126,7 @@ function parse(argv: string[]) { options: { config: { type: 'string' }, source: { type: 'string', multiple: true }, + crawl: { type: 'string' }, 'base-url': { type: 'string' }, index: { type: 'string' }, 'dry-run': { type: 'boolean' }, @@ -158,7 +160,10 @@ async function runIngest(values: Values, io: Io): Promise { const loaded = await loadConfig(io.cwd, values.config); try { const target = indexConfig(loaded); - const sources = values.source?.length ? values.source.map((source) => toSource(source, values['base-url'], io.cwd)) : loaded.config.ingest?.sources; + const cliSources: IngestSource[] = []; + if (values.crawl) cliSources.push({ type: 'crawl', url: values.crawl }); + if (values.source?.length) cliSources.push(...values.source.map((source) => toSource(source, values['base-url'], io.cwd))); + const sources = cliSources.length ? cliSources : loaded.config.ingest?.sources; if (!sources?.length) throw new ConfigError('Nothing to ingest: pass --source, or set ingest.sources in the config.'); // A model only runs with --contextualize, so a run never spends on one by surprise: the config holds which. @@ -221,6 +226,9 @@ async function runSearch(question: string, values: Values, io: Io): Promise { + site.set('/crawl/cli/start', '

Start

Start with Sub

'); + site.set('/crawl/cli/sub', '

Sub

Sub content

'); + const cwd = project({ shared: 'cli-crawl' }); + try { + const crawlFlag = await run(['ingest', '--crawl', `${site.url}/crawl/cli/start`], cwd); + expect(crawlFlag.code).toBe(EXIT.ok); + expect(crawlFlag.stdout).toContain('Indexed 2 of 2 pages'); + + const crawlSource = await run(['ingest', '--source', `crawl:${site.url}/crawl/cli/start`, '--force'], cwd); + expect(crawlSource.code).toBe(EXIT.ok); + expect(crawlSource.stdout).toContain('Indexed 2 of 2 pages'); + } finally { + for (const p of ['/crawl/cli/start', '/crawl/cli/sub']) site.remove(p); + } + }); }); describe('toSource', () => { it('tells sitemaps, pages and folders apart', () => { + expect(toSource('crawl:https://acme.test/start', undefined, '/x')).toEqual({ type: 'crawl', url: 'https://acme.test/start' }); expect(toSource('https://acme.test/sitemap.xml', undefined, '/x')).toEqual({ type: 'sitemap', url: 'https://acme.test/sitemap.xml' }); expect(toSource('https://acme.test/sitemap.xml.gz?v=2', undefined, '/x')).toEqual({ type: 'sitemap', url: 'https://acme.test/sitemap.xml.gz?v=2' }); expect(toSource('https://acme.test/pricing', undefined, '/x')).toEqual({ type: 'urls', urls: ['https://acme.test/pricing'] }); diff --git a/packages/ingest/src/discover.ts b/packages/ingest/src/discover.ts index c7ca3969..bf617aad 100644 --- a/packages/ingest/src/discover.ts +++ b/packages/ingest/src/discover.ts @@ -1,6 +1,7 @@ import { readdir } from 'node:fs/promises'; import { join, relative, sep } from 'node:path'; import { gunzipSync } from 'node:zlib'; +import { robotsFor, type Robots } from './robots.js'; import type { DiscoveredPage, IngestSource } from './types.js'; /** Sitemap indexes can point at more indexes; this many levels is plenty. */ @@ -13,6 +14,10 @@ export interface DiscoverOptions { timeoutMs: number; include?: string[]; exclude?: string[]; + /** Pages fetched at once during crawl. Default 4. */ + concurrency?: number; + /** Obey robots.txt during crawl. Default true. */ + respectRobots?: boolean; } export interface Discovery { @@ -35,6 +40,8 @@ export async function discover(sources: IngestSource[], options: DiscoverOptions for (const url of source.urls) add({ url }); } else if (source.type === 'sitemap') { await readSitemap(source.url, options, 0, new Set(), add, errors); + } else if (source.type === 'crawl') { + await crawlSite(source, options, add, errors); } else { try { for (const page of await readDirectory(source.path, source.baseUrl)) add(page); @@ -103,6 +110,190 @@ async function readDirectory(path: string, baseUrl: string): Promise a.url.localeCompare(b.url)); } +const DEFAULT_CRAWL_DEPTH = 3; +const DEFAULT_CRAWL_PAGES = 500; + +/** + * Crawl a site breadth-first from `url`, following same-origin `` + * links. `maxDepth` is link hops from the start (depth 0); the start counts + * toward `maxPages`. Other origins, `mailto:`, `tel:`, fragment-only links + * and non-HTML responses are skipped. Robots.txt is respected when + * `respectRobots` is not false. Already-fetched HTML is stored on each page + * so extraction doesn't fetch again. + */ +async function crawlSite( + source: Extract, + options: DiscoverOptions, + add: (page: DiscoveredPage) => void, + errors: string[] +): Promise { + const maxDepth = source.maxDepth ?? DEFAULT_CRAWL_DEPTH; + const maxPages = source.maxPages ?? DEFAULT_CRAWL_PAGES; + if (maxPages <= 0) return; + + const startUrl = normalizeUrl(source.url); + if (!startUrl) { + errors.push(`${source.url}: not a valid URL`); + return; + } + + // Apply existing include/exclude rules before fetching, including the starting URL. + if (!matches(startUrl, options.include, options.exclude)) { + return; + } + + const origin = new URL(startUrl).origin; + const respectRobots = options.respectRobots !== false; + const concurrency = Math.max(1, options.concurrency ?? 4); + + const policies = new Map>(); + async function getRobots(urlOrOrigin: string): Promise { + if (!respectRobots) return undefined; + const fetchWithPolicy = options.fetch as typeof fetch & { policy?: (url: string) => Promise }; + if (typeof fetchWithPolicy.policy === 'function') { + return fetchWithPolicy.policy(urlOrOrigin); + } + const o = new URL(urlOrOrigin).origin; + let pending = policies.get(o); + if (!pending) { + pending = robotsFor(o, ((input, init) => + options.fetch(String(input), { + ...init, + headers: { 'user-agent': options.userAgent, ...(init?.headers as Record) }, + signal: AbortSignal.timeout(options.timeoutMs), + })) as typeof fetch).then((rules) => { + if (rules.unavailable) errors.push(`${o}/robots.txt unavailable (${rules.unavailable})`); + return rules; + }); + policies.set(o, pending); + } + return pending; + } + + let robots: Robots | undefined; + if (respectRobots) { + try { + robots = await getRobots(startUrl); + } catch { + robots = undefined; + } + if (robots && !robots.allowed(startUrl)) { + return; + } + } + + const seen = new Set(); + seen.add(startUrl); + + let currentLevel = [startUrl]; + let depth = 0; + let count = 0; + + while (currentLevel.length > 0 && depth <= maxDepth && count < maxPages) { + const nextLevel: string[] = []; + let nextIndex = 0; + let inFlight = 0; + const waiters: Array<() => void> = []; + const notifyWaiters = () => { + while (waiters.length > 0) waiters.shift()?.(); + }; + + const isLastDepth = depth >= maxDepth; + + async function worker(): Promise { + while (true) { + if (count >= maxPages) return; + if (nextIndex >= currentLevel.length) return; + + if (count + inFlight >= maxPages) { + if (inFlight === 0) return; + await new Promise((resolve) => waiters.push(resolve)); + continue; + } + + const url = currentLevel[nextIndex++]; + inFlight++; + + try { + if (robots && !robots.allowed(url)) continue; + + let response: Response; + try { + response = await options.fetch(url, { + headers: { 'user-agent': options.userAgent, accept: 'text/html,application/xhtml+xml' }, + signal: AbortSignal.timeout(options.timeoutMs), + redirect: 'follow', + }); + } catch (error) { + errors.push(`${url}: ${message(error)}`); + continue; + } + + if (!response.ok) { + errors.push(`${url}: HTTP ${response.status}`); + continue; + } + + const type = response.headers.get('content-type') ?? ''; + if (type && !/html/i.test(type)) continue; + + const html = await response.text(); + count++; + add({ url, html }); + + if (!isLastDepth) { + for (const href of extractLinks(html, url, origin)) { + const normalized = normalizeUrl(href); + if (!normalized || seen.has(normalized)) continue; + if (!matches(normalized, options.include, options.exclude)) continue; + if (robots && !robots.allowed(normalized)) continue; + seen.add(normalized); + nextLevel.push(normalized); + } + } + } finally { + inFlight--; + notifyWaiters(); + } + } + } + + const workerCount = Math.min(concurrency, currentLevel.length); + await Promise.all(Array.from({ length: workerCount }, worker)); + + currentLevel = nextLevel; + depth++; + } +} + +/** Same-origin `` links on a page, resolved and filtered. */ +function extractLinks(html: string, baseUrl: string, origin: string): string[] { + const links: string[] = []; + const anchorRegex = /]*?(?<=\s)href\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))[^>]*>/gi; + for (const match of html.matchAll(anchorRegex)) { + const raw = match[1] ?? match[2] ?? match[3]; + if (!raw) continue; + const clean = raw.replace(/&/g, '&').trim(); + if (!clean) continue; + // Skip mailto:, tel:, and other non-http(s) schemes. + if (/^(mailto|tel|javascript|data|blob):/i.test(clean)) continue; + // Skip fragment-only links. + if (clean.startsWith('#')) continue; + + let resolved: URL; + try { + resolved = new URL(clean, baseUrl); + } catch { + continue; + } + if (resolved.protocol !== 'http:' && resolved.protocol !== 'https:') continue; + // Same origin only. + if (resolved.origin !== origin) continue; + links.push(resolved.href); + } + return links; +} + /** The URL without its fragment, or undefined when it isn't http(s). */ export function normalizeUrl(url: string): string | undefined { try { diff --git a/packages/ingest/src/ingest.ts b/packages/ingest/src/ingest.ts index f47f98b6..ea01439a 100644 --- a/packages/ingest/src/ingest.ts +++ b/packages/ingest/src/ingest.ts @@ -79,11 +79,14 @@ export async function ingest(config: IngestConfig): Promise { return pending; } - const send: typeof fetch = async (input, init) => { - const url = String(input); - const rules = config.respectRobots === false ? undefined : await policy(url); - return scheduled(url, init, rules?.crawlDelay); - }; + const send: typeof fetch = Object.assign( + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + const rules = config.respectRobots === false ? undefined : await policy(url); + return scheduled(url, init, rules?.crawlDelay); + }, + { policy } + ); if (embedder.dimensions !== store.dimensions) { throw new Error( @@ -102,7 +105,15 @@ export async function ingest(config: IngestConfig): Promise { ); } - const found = await discover(config.sources, { fetch: send, userAgent, timeoutMs, include: config.include, exclude: config.exclude }); + const found = await discover(config.sources, { + fetch: send, + userAgent, + timeoutMs, + include: config.include, + exclude: config.exclude, + concurrency: config.concurrency, + respectRobots: config.respectRobots, + }); if (config.maxPages !== undefined && config.maxPages >= 0 && found.pages.length > config.maxPages) { found.pages = found.pages.slice(0, Math.floor(config.maxPages)); // Pages left out aren't removed pages, so pruning a partial run isn't safe either. @@ -135,6 +146,10 @@ export async function ingest(config: IngestConfig): Promise { const doc = /\.html?$/i.test(page.file) ? extractHtml(source, page.url, selectorFor(page.url)) : extractMarkdown(source, page.url); return { kind: 'page', doc }; } + if (page.html !== undefined) { + const doc = extractHtml(page.html, page.url, selectorFor(page.url)); + return { kind: 'page', doc }; + } if (config.respectRobots !== false) { const rules = await policy(page.url); if (rules.unavailable) return { kind: 'skip', reason: `robots.txt unavailable (${rules.unavailable})` }; diff --git a/packages/ingest/src/types.ts b/packages/ingest/src/types.ts index bc12b59c..bfe48164 100644 --- a/packages/ingest/src/types.ts +++ b/packages/ingest/src/types.ts @@ -6,7 +6,9 @@ export type IngestSource = /** Pages listed by hand. */ | { type: 'urls'; urls: string[] } /** Local `.md`, `.mdx` and `.html` files, published under `baseUrl`. */ - | { type: 'directory'; path: string; baseUrl: string }; + | { type: 'directory'; path: string; baseUrl: string } + /** Crawl a site starting at `url`, following same-origin links. */ + | { type: 'crawl'; url: string; maxDepth?: number; maxPages?: number }; export interface IngestConfig { /** Names the index, so a new one can be built next to the old. Default `default`. */ @@ -67,6 +69,8 @@ export interface DiscoveredPage { url: string; /** For directory sources: the file to read instead of fetching. */ file?: string; + /** For crawl sources: the HTML already fetched during discovery, so extraction doesn't fetch again. */ + html?: string; } export interface ExtractedDoc { diff --git a/packages/ingest/tests/ingest.test.ts b/packages/ingest/tests/ingest.test.ts index 07f8f0b5..45e68677 100644 --- a/packages/ingest/tests/ingest.test.ts +++ b/packages/ingest/tests/ingest.test.ts @@ -75,6 +75,118 @@ describe('discover', () => { } }); + it('crawls breadth-first and respects maxDepth', async () => { + site.set('/crawl/bfs/start', 'AB'); + site.set('/crawl/bfs/a', 'C'); + site.set('/crawl/bfs/b', 'D'); + site.set('/crawl/bfs/c', 'E'); + site.set('/crawl/bfs/d', '

D

'); + site.set('/crawl/bfs/e', '

E

'); + try { + const depth0 = await discover([{ type: 'crawl', url: `${site.url}/crawl/bfs/start`, maxDepth: 0 }], discoverOptions()); + expect(depth0.pages.map((p) => p.url.replace(site.url, ''))).toEqual(['/crawl/bfs/start']); + + const depth1 = await discover([{ type: 'crawl', url: `${site.url}/crawl/bfs/start`, maxDepth: 1 }], discoverOptions()); + expect(depth1.pages.map((p) => p.url.replace(site.url, ''))).toEqual(['/crawl/bfs/start', '/crawl/bfs/a', '/crawl/bfs/b']); + + const depth2 = await discover([{ type: 'crawl', url: `${site.url}/crawl/bfs/start`, maxDepth: 2 }], discoverOptions()); + expect(depth2.pages.map((p) => p.url.replace(site.url, ''))).toEqual([ + '/crawl/bfs/start', + '/crawl/bfs/a', + '/crawl/bfs/b', + '/crawl/bfs/c', + '/crawl/bfs/d', + ]); + } finally { + for (const p of ['/crawl/bfs/start', '/crawl/bfs/a', '/crawl/bfs/b', '/crawl/bfs/c', '/crawl/bfs/d', '/crawl/bfs/e']) site.remove(p); + } + }); + + it('limits crawl with maxPages counting the start page', async () => { + site.set('/crawl/mp/start', 'ABC'); + site.set('/crawl/mp/a', '

A

'); + site.set('/crawl/mp/b', '

B

'); + site.set('/crawl/mp/c', '

C

'); + try { + const res = await discover([{ type: 'crawl', url: `${site.url}/crawl/mp/start`, maxPages: 2 }], discoverOptions()); + expect(res.pages).toHaveLength(2); + expect(res.pages[0].url).toBe(`${site.url}/crawl/mp/start`); + } finally { + for (const p of ['/crawl/mp/start', '/crawl/mp/a', '/crawl/mp/b', '/crawl/mp/c']) site.remove(p); + } + }); + + it('skips cross-origin links, mailto, tel, fragments, and non-HTML responses while deduplicating', async () => { + site.set( + '/crawl/filter/start', + `External + Mail + Tel + Top + Section + Target with frag + Target with frag2 + Target bare + PDF file` + ); + site.set('/crawl/filter/target', '

Target page

'); + const before = site.requests.length; + try { + const res = await discover([{ type: 'crawl', url: `${site.url}/crawl/filter/start` }], discoverOptions()); + expect(res.pages.map((p) => p.url.replace(site.url, ''))).toEqual(['/crawl/filter/start', '/crawl/filter/target']); + const requests = site.requests.slice(before); + expect(requests.filter((r) => r === '/crawl/filter/target')).toHaveLength(1); + expect(requests).toContain('/files/manual.pdf'); + expect(requests).not.toContain('/other'); + } finally { + for (const p of ['/crawl/filter/start', '/crawl/filter/target']) site.remove(p); + } + }); + + it('applies exclude rules before fetching, including the starting URL', async () => { + site.set('/crawl/ex/start', 'KeepSkip'); + site.set('/crawl/ex/keep', '

Keep

'); + site.set('/crawl/ex/skip', '

Skip

'); + const before = site.requests.length; + try { + const res = await discover([{ type: 'crawl', url: `${site.url}/crawl/ex/start` }], { + ...discoverOptions(), + exclude: ['**/skip'], + }); + expect(res.pages.map((p) => p.url.replace(site.url, ''))).toEqual(['/crawl/ex/start', '/crawl/ex/keep']); + expect(site.requests.slice(before)).not.toContain('/crawl/ex/skip'); + + const startExcluded = await discover([{ type: 'crawl', url: `${site.url}/crawl/ex/skip` }], { + ...discoverOptions(), + exclude: ['**/skip'], + }); + expect(startExcluded.pages).toEqual([]); + expect(site.requests.slice(before)).not.toContain('/crawl/ex/skip'); + } finally { + for (const p of ['/crawl/ex/start', '/crawl/ex/keep', '/crawl/ex/skip']) site.remove(p); + } + }); + + it('respects robots disallow rules during crawl', async () => { + site.set('/robots.txt', 'User-agent: *\nDisallow: /crawl/rb/blocked', 'text/plain'); + site.set('/crawl/rb/start', 'AllowedBlocked'); + site.set('/crawl/rb/allowed', '

Allowed

'); + site.set('/crawl/rb/blocked', '

Blocked

'); + const before = site.requests.length; + try { + const res = await discover([{ type: 'crawl', url: `${site.url}/crawl/rb/start` }], discoverOptions()); + expect(res.pages.map((p) => p.url.replace(site.url, ''))).toEqual(['/crawl/rb/start', '/crawl/rb/allowed']); + expect(site.requests.slice(before)).not.toContain('/crawl/rb/blocked'); + + // Start page blocked terminates crawl immediately + const blockedStart = await discover([{ type: 'crawl', url: `${site.url}/crawl/rb/blocked` }], discoverOptions()); + expect(blockedStart.pages).toEqual([]); + } finally { + site.remove('/robots.txt'); + for (const p of ['/crawl/rb/start', '/crawl/rb/allowed', '/crawl/rb/blocked']) site.remove(p); + } + }); + it('turns globs into anchored patterns', () => { expect(globToRegExp('https://acme.test/docs/*').test('https://acme.test/docs/a')).toBe(true); expect(globToRegExp('https://acme.test/docs/*').test('https://acme.test/docs/a/b')).toBe(false); @@ -327,4 +439,30 @@ describe('ingest', () => { rmSync(dir, { recursive: true, force: true }); } }); + + it('crawls and extracts pages with exactly one page fetch per URL and reuses cached HTML', async () => { + const pageHtml = (title: string, body: string) => + `${title}

${title}

${body}

`; + site.set('/crawl/ingest/start', pageHtml('Start', 'Start page with Link A and Link B.')); + site.set('/crawl/ingest/a', pageHtml('Page A', 'Content for page A with valuable info.')); + site.set('/crawl/ingest/b', pageHtml('Page B', 'Content for page B with useful details.')); + const before = site.requests.length; + const { config } = setup({ sources: [{ type: 'crawl', url: `${site.url}/crawl/ingest/start` }] }); + try { + const stats = await ingest(config); + expect(stats.indexed).toBe(3); + const requests = site.requests.slice(before); + // Exactly one fetch per page URL across discovery + extraction + expect(requests.filter((r) => r === '/crawl/ingest/start')).toHaveLength(1); + expect(requests.filter((r) => r === '/crawl/ingest/a')).toHaveLength(1); + expect(requests.filter((r) => r === '/crawl/ingest/b')).toHaveLength(1); + + // Second run: contentHash matches, everything unchanged, 0 embeds + const rerun = await ingest(config); + expect(rerun.unchanged).toBe(3); + expect(rerun.embedded).toBe(0); + } finally { + for (const p of ['/crawl/ingest/start', '/crawl/ingest/a', '/crawl/ingest/b']) site.remove(p); + } + }); }); diff --git a/website/src/content/docs/core/knowledge-base.mdx b/website/src/content/docs/core/knowledge-base.mdx index f25103e8..b3f0fc6b 100644 --- a/website/src/content/docs/core/knowledge-base.mdx +++ b/website/src/content/docs/core/knowledge-base.mdx @@ -144,6 +144,7 @@ Network sources respect `robots.txt` by default, using the `ChatCopsBot` group o | `{ type: 'sitemap', url }` | A sitemap, a sitemap index, or a gzipped one (`.xml.gz`) | | `{ type: 'urls', urls }` | The pages you list | | `{ type: 'directory', path, baseUrl }` | Local `.md`, `.mdx` and `.html` files, cited as if published under `baseUrl` | +| `{ type: 'crawl', url, maxDepth?, maxPages? }` | Crawl a site breadth-first from `url`, following same-origin links (defaults: `maxDepth: 3`, `maxPages: 500`) | The rest of the config: From 758a649b8d90b0540f4d2c1e9cbde77b56e17a88 Mon Sep 17 00:00:00 2001 From: bhushan4work Date: Fri, 6 Nov 2026 00:51:25 +0530 Subject: [PATCH 2/3] fix(ingest,cli): address review feedback for crawl discovery (#586) --- api/ingest.api.md | 1 + packages/cli/README.md | 2 +- packages/cli/src/main.ts | 2 +- packages/cli/tests/cli.test.ts | 16 ++ packages/ingest/src/discover.ts | 211 ++++++++++++++--- packages/ingest/src/ingest.ts | 5 +- packages/ingest/tests/ingest.test.ts | 213 ++++++++++++++++++ .../src/content/docs/core/knowledge-base.mdx | 2 +- 8 files changed, 420 insertions(+), 32 deletions(-) diff --git a/api/ingest.api.md b/api/ingest.api.md index 98afb906..f93bc3c9 100644 --- a/api/ingest.api.md +++ b/api/ingest.api.md @@ -38,6 +38,7 @@ export interface DiscoverOptions { exclude?: string[]; concurrency?: number; respectRobots?: boolean; + maxPages?: number; } export interface Discovery { diff --git a/packages/cli/README.md b/packages/cli/README.md index a907b5d4..71519fe8 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -25,7 +25,7 @@ export default defineConfig({ }); ``` -`chatcops ingest` options: `--source` (repeatable; a sitemap, a page URL, or a folder with `--base-url`), `--index`, `--dry-run`, `--max-pages`, `--concurrency`, `--prune`, `--force`, `--json`. `chatcops search` options: `--index`, `--top-k`, `--json`. Run `chatcops --help` for details. +`chatcops ingest` options: `--source` (repeatable; a sitemap, a page URL, or a folder with `--base-url`), `--crawl`, `--index`, `--dry-run`, `--max-pages`, `--concurrency`, `--prune`, `--force`, `--json`. `chatcops search` options: `--index`, `--top-k`, `--json`. Run `chatcops --help` for details. `chatcops test` runs eval files (`evals/**/*.chat.yaml` by default) against the bot in the config's `server` section: diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 5b03c854..63c40d6d 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -164,7 +164,7 @@ async function runIngest(values: Values, io: Io): Promise { if (values.crawl) cliSources.push({ type: 'crawl', url: values.crawl }); if (values.source?.length) cliSources.push(...values.source.map((source) => toSource(source, values['base-url'], io.cwd))); const sources = cliSources.length ? cliSources : loaded.config.ingest?.sources; - if (!sources?.length) throw new ConfigError('Nothing to ingest: pass --source, or set ingest.sources in the config.'); + if (!sources?.length) throw new ConfigError('Nothing to ingest: pass --source, --crawl, or set ingest.sources in the config.'); // A model only runs with --contextualize, so a run never spends on one by surprise: the config holds which. if (values.contextualize && !loaded.config.ingest?.contextualize) { diff --git a/packages/cli/tests/cli.test.ts b/packages/cli/tests/cli.test.ts index 32b92cd9..f1deea2a 100644 --- a/packages/cli/tests/cli.test.ts +++ b/packages/cli/tests/cli.test.ts @@ -251,6 +251,22 @@ export default { expect(await run(['ingest', '--source', './docs'], cwd)).toMatchObject({ code: EXIT.unusable, stderr: expect.stringContaining('needs --base-url') }); }); + it('errors when nothing to ingest without source or crawl flags', async () => { + const cwd = project({ + shared: 'no-sources', + body: `import { memoryStorage, memoryVectors } from '@chatcops/core'; +export default { + storage: { ...memoryStorage(), vectors: memoryVectors({ dimensions: 4 }) }, + embedder: { id: 'test:fake@4', dimensions: 4, async embed() { return []; } }, + ingest: {}, +};`, + }); + expect(await run(['ingest'], cwd)).toMatchObject({ + code: EXIT.unusable, + stderr: expect.stringContaining('Nothing to ingest: pass --source, --crawl, or set ingest.sources in the config.'), + }); + }); + it('ingests with --crawl and crawl: source', async () => { site.set('/crawl/cli/start', '

Start

Start with Sub

'); site.set('/crawl/cli/sub', '

Sub

Sub content

'); diff --git a/packages/ingest/src/discover.ts b/packages/ingest/src/discover.ts index bf617aad..05a89f07 100644 --- a/packages/ingest/src/discover.ts +++ b/packages/ingest/src/discover.ts @@ -18,6 +18,8 @@ export interface DiscoverOptions { concurrency?: number; /** Obey robots.txt during crawl. Default true. */ respectRobots?: boolean; + /** Limit total pages discovered. */ + maxPages?: number; } export interface Discovery { @@ -73,7 +75,10 @@ async function readSitemap( headers: { 'user-agent': options.userAgent }, signal: AbortSignal.timeout(options.timeoutMs), }); - if (!response.ok) throw new Error(`HTTP ${response.status}`); + if (!response.ok) { + await cancelBody(response); + throw new Error(`HTTP ${response.status}`); + } const bytes = new Uint8Array(await response.arrayBuffer()); // gzip's magic number; some servers send .xml.gz without saying so. xml = (bytes[0] === 0x1f && bytes[1] === 0x8b ? gunzipSync(bytes) : Buffer.from(bytes)).toString('utf8'); @@ -113,6 +118,14 @@ async function readDirectory(path: string, baseUrl: string): Promise { + try { + await response.body?.cancel(); + } catch { + // A failed cleanup should not prevent crawling from continuing. + } +} + /** * Crawl a site breadth-first from `url`, following same-origin `` * links. `maxDepth` is link hops from the start (depth 0); the start counts @@ -128,8 +141,14 @@ async function crawlSite( errors: string[] ): Promise { const maxDepth = source.maxDepth ?? DEFAULT_CRAWL_DEPTH; - const maxPages = source.maxPages ?? DEFAULT_CRAWL_PAGES; - if (maxPages <= 0) return; + let effectiveMaxPages = source.maxPages ?? DEFAULT_CRAWL_PAGES; + if (options.maxPages !== undefined && options.maxPages >= 0) { + effectiveMaxPages = Math.min(effectiveMaxPages, Math.floor(options.maxPages)); + } + if (effectiveMaxPages <= 0) { + errors.push(`stopped at --max-pages ${effectiveMaxPages}`); + return; + } const startUrl = normalizeUrl(source.url); if (!startUrl) { @@ -139,12 +158,12 @@ async function crawlSite( // Apply existing include/exclude rules before fetching, including the starting URL. if (!matches(startUrl, options.include, options.exclude)) { + errors.push(`${startUrl}: does not match include/exclude rules`); return; } - const origin = new URL(startUrl).origin; - const respectRobots = options.respectRobots !== false; const concurrency = Math.max(1, options.concurrency ?? 4); + const respectRobots = options.respectRobots !== false; const policies = new Map>(); async function getRobots(urlOrOrigin: string): Promise { @@ -170,27 +189,97 @@ async function crawlSite( return pending; } + let startResponse: Response; + try { + startResponse = await options.fetch(startUrl, { + headers: { 'user-agent': options.userAgent, accept: 'text/html,application/xhtml+xml' }, + signal: AbortSignal.timeout(options.timeoutMs), + redirect: 'follow', + }); + } catch (error) { + errors.push(`${startUrl}: ${message(error)}`); + return; + } + + const canonicalStartUrl = normalizeUrl(startResponse.url || startUrl) ?? (startResponse.url || startUrl); + + if (!startResponse.ok) { + await cancelBody(startResponse); + errors.push(`${canonicalStartUrl}: HTTP ${startResponse.status}`); + return; + } + + const startType = startResponse.headers.get('content-type') ?? ''; + if (startType && !/html/i.test(startType)) { + await cancelBody(startResponse); + const mediaType = startType.split(';')[0].trim(); + errors.push(`${canonicalStartUrl}: not HTML${mediaType ? ` (${mediaType})` : ''}`); + return; + } + + if (!matches(canonicalStartUrl, options.include, options.exclude)) { + await cancelBody(startResponse); + errors.push(`${canonicalStartUrl}: does not match include/exclude rules`); + return; + } + + const origin = new URL(canonicalStartUrl).origin; + let robots: Robots | undefined; if (respectRobots) { try { - robots = await getRobots(startUrl); + robots = await getRobots(canonicalStartUrl); } catch { robots = undefined; } - if (robots && !robots.allowed(startUrl)) { + if (robots && !robots.allowed(canonicalStartUrl)) { + await cancelBody(startResponse); + errors.push(`${canonicalStartUrl}: disallowed by robots.txt`); return; } } + let startHtml: string; + try { + startHtml = await startResponse.text(); + } catch (error) { + errors.push(`${canonicalStartUrl}: ${message(error)}`); + return; + } + const seen = new Set(); seen.add(startUrl); + seen.add(canonicalStartUrl); + + add({ url: canonicalStartUrl, html: startHtml }); + let count = 1; + + let currentLevel: string[] = []; + if (maxDepth > 0) { + for (const href of extractLinks(startHtml, canonicalStartUrl, origin)) { + const normalized = normalizeUrl(href); + if (!normalized || seen.has(normalized)) continue; + if (!matches(normalized, options.include, options.exclude)) continue; + if (robots && !robots.allowed(normalized)) continue; + seen.add(normalized); + currentLevel.push(normalized); + } + } - let currentLevel = [startUrl]; - let depth = 0; - let count = 0; + if (count >= effectiveMaxPages) { + if (currentLevel.length > 0) { + errors.push(`stopped at --max-pages ${effectiveMaxPages}`); + } + return; + } - while (currentLevel.length > 0 && depth <= maxDepth && count < maxPages) { + let depth = 1; + while (currentLevel.length > 0 && depth <= maxDepth && count < effectiveMaxPages) { const nextLevel: string[] = []; + const isLastDepth = depth >= maxDepth; + const pagesByPosition: Array = new Array(currentLevel.length); + const linksByPosition: string[][] = Array.from({ length: currentLevel.length }, () => []); + let nextIndex = 0; let inFlight = 0; const waiters: Array<() => void> = []; @@ -198,20 +287,19 @@ async function crawlSite( while (waiters.length > 0) waiters.shift()?.(); }; - const isLastDepth = depth >= maxDepth; - async function worker(): Promise { while (true) { - if (count >= maxPages) return; + if (count >= effectiveMaxPages) return; if (nextIndex >= currentLevel.length) return; - if (count + inFlight >= maxPages) { + if (count + inFlight >= effectiveMaxPages) { if (inFlight === 0) return; await new Promise((resolve) => waiters.push(resolve)); continue; } - const url = currentLevel[nextIndex++]; + const index = nextIndex++; + const url = currentLevel[index]; inFlight++; try { @@ -229,27 +317,67 @@ async function crawlSite( continue; } + const canonicalUrl = normalizeUrl(response.url || url) ?? (response.url || url); + if (!response.ok) { - errors.push(`${url}: HTTP ${response.status}`); + await cancelBody(response); + errors.push(`${canonicalUrl}: HTTP ${response.status}`); + continue; + } + + let finalOrigin: string; + try { + finalOrigin = new URL(canonicalUrl).origin; + } catch { + await cancelBody(response); + continue; + } + + // Skip subsequent pages whose final redirected URL is outside the established crawl origin. + if (finalOrigin !== origin) { + await cancelBody(response); + seen.add(canonicalUrl); + continue; + } + + // If same-origin redirect target was already seen, skip. + if (canonicalUrl !== url && seen.has(canonicalUrl)) { + await cancelBody(response); + continue; + } + + if (!matches(canonicalUrl, options.include, options.exclude)) { + await cancelBody(response); + seen.add(canonicalUrl); + continue; + } + + if (robots && !robots.allowed(canonicalUrl)) { + await cancelBody(response); + seen.add(canonicalUrl); continue; } const type = response.headers.get('content-type') ?? ''; - if (type && !/html/i.test(type)) continue; + if (type && !/html/i.test(type)) { + await cancelBody(response); + continue; + } + + let html: string; + try { + html = await response.text(); + } catch (error) { + errors.push(`${canonicalUrl}: ${message(error)}`); + continue; + } - const html = await response.text(); + seen.add(canonicalUrl); + pagesByPosition[index] = { url: canonicalUrl, html }; count++; - add({ url, html }); if (!isLastDepth) { - for (const href of extractLinks(html, url, origin)) { - const normalized = normalizeUrl(href); - if (!normalized || seen.has(normalized)) continue; - if (!matches(normalized, options.include, options.exclude)) continue; - if (robots && !robots.allowed(normalized)) continue; - seen.add(normalized); - nextLevel.push(normalized); - } + linksByPosition[index] = extractLinks(html, canonicalUrl, origin); } } finally { inFlight--; @@ -261,8 +389,35 @@ async function crawlSite( const workerCount = Math.min(concurrency, currentLevel.length); await Promise.all(Array.from({ length: workerCount }, worker)); + for (let i = 0; i < currentLevel.length; i++) { + const page = pagesByPosition[i]; + if (page) { + add(page); + } + if (!isLastDepth) { + for (const href of linksByPosition[i]) { + const normalized = normalizeUrl(href); + if (!normalized || seen.has(normalized)) continue; + if (!matches(normalized, options.include, options.exclude)) continue; + if (robots && !robots.allowed(normalized)) continue; + seen.add(normalized); + nextLevel.push(normalized); + } + } + } + + const previousLength = currentLevel.length; currentLevel = nextLevel; depth++; + + if (count >= effectiveMaxPages) { + if (nextIndex < previousLength || currentLevel.length > 0) { + if (!errors.includes(`stopped at --max-pages ${effectiveMaxPages}`)) { + errors.push(`stopped at --max-pages ${effectiveMaxPages}`); + } + } + break; + } } } diff --git a/packages/ingest/src/ingest.ts b/packages/ingest/src/ingest.ts index ea01439a..c9ef19d0 100644 --- a/packages/ingest/src/ingest.ts +++ b/packages/ingest/src/ingest.ts @@ -113,11 +113,14 @@ export async function ingest(config: IngestConfig): Promise { exclude: config.exclude, concurrency: config.concurrency, respectRobots: config.respectRobots, + maxPages: config.maxPages, }); if (config.maxPages !== undefined && config.maxPages >= 0 && found.pages.length > config.maxPages) { found.pages = found.pages.slice(0, Math.floor(config.maxPages)); // Pages left out aren't removed pages, so pruning a partial run isn't safe either. - found.errors.push(`stopped at --max-pages ${config.maxPages}`); + if (!found.errors.includes(`stopped at --max-pages ${config.maxPages}`)) { + found.errors.push(`stopped at --max-pages ${config.maxPages}`); + } } config.onEvent?.({ type: 'discovered', pages: found.pages.length, errors: found.errors }); errors.push(...found.errors); diff --git a/packages/ingest/tests/ingest.test.ts b/packages/ingest/tests/ingest.test.ts index 45e68677..de461b0d 100644 --- a/packages/ingest/tests/ingest.test.ts +++ b/packages/ingest/tests/ingest.test.ts @@ -107,10 +107,18 @@ describe('discover', () => { site.set('/crawl/mp/a', '

A

'); site.set('/crawl/mp/b', '

B

'); site.set('/crawl/mp/c', '

C

'); + const before = site.requests.length; try { const res = await discover([{ type: 'crawl', url: `${site.url}/crawl/mp/start`, maxPages: 2 }], discoverOptions()); expect(res.pages).toHaveLength(2); expect(res.pages[0].url).toBe(`${site.url}/crawl/mp/start`); + expect(res.pages[1].url).toBe(`${site.url}/crawl/mp/a`); + expect(res.errors).toEqual(['stopped at --max-pages 2']); + const requests = site.requests.slice(before); + expect(requests).toContain('/crawl/mp/start'); + expect(requests).toContain('/crawl/mp/a'); + expect(requests).not.toContain('/crawl/mp/b'); + expect(requests).not.toContain('/crawl/mp/c'); } finally { for (const p of ['/crawl/mp/start', '/crawl/mp/a', '/crawl/mp/b', '/crawl/mp/c']) site.remove(p); } @@ -187,6 +195,176 @@ describe('discover', () => { } }); + it('filters cross-origin links without requesting other origin and leaves errors empty', async () => { + const other = await fixtureSite(); + site.set('/crawl/cross/start', `OtherLocal`); + site.set('/crawl/cross/local', '

Local page

'); + other.set('/external', '

External page

'); + try { + const res = await discover([{ type: 'crawl', url: `${site.url}/crawl/cross/start` }], discoverOptions()); + expect(res.pages.map((p) => p.url.replace(site.url, ''))).toEqual(['/crawl/cross/start', '/crawl/cross/local']); + expect(other.requests).toEqual([]); + expect(res.errors).toEqual([]); + } finally { + await other.close(); + for (const p of ['/crawl/cross/start', '/crawl/cross/local']) site.remove(p); + } + }); + + it('establishes origin and robots policy from homepage redirect to another origin', async () => { + const targetSite = await fixtureSite(); + targetSite.set('/robots.txt', 'User-agent: *\nDisallow: /crawl/blocked-on-target', 'text/plain'); + site.set('/crawl/home', '', 'text/plain', 302, { location: `${targetSite.url}/crawl/landing` }); + targetSite.set( + '/crawl/landing', + 'SubpageBlockedOffsite' + ); + targetSite.set('/crawl/sub', '

Subpage content

'); + targetSite.set('/crawl/blocked-on-target', '

Blocked content

'); + try { + const res = await discover([{ type: 'crawl', url: `${site.url}/crawl/home` }], discoverOptions()); + expect(res.pages.map((p) => p.url.replace(targetSite.url, ''))).toEqual(['/crawl/landing', '/crawl/sub']); + expect(targetSite.requests).not.toContain('/crawl/blocked-on-target'); + expect(res.errors).toEqual([]); + } finally { + site.remove('/crawl/home'); + await targetSite.close(); + } + }); + + it('resolves relative links against canonical URL after trailing-slash redirect', async () => { + site.set('/crawl/docs', '', 'text/plain', 301, { location: `${site.url}/crawl/docs/` }); + site.set('/crawl/docs/', 'Guide'); + site.set('/crawl/docs/guide', '

Guide content

'); + site.set('/crawl/guide', '

Root guide

'); + try { + const res = await discover([{ type: 'crawl', url: `${site.url}/crawl/docs` }], discoverOptions()); + expect(res.pages.map((p) => p.url.replace(site.url, ''))).toEqual(['/crawl/docs/', '/crawl/docs/guide']); + expect(res.errors).toEqual([]); + expect(site.requests).not.toContain('/crawl/guide'); + } finally { + for (const p of ['/crawl/docs', '/crawl/docs/', '/crawl/docs/guide', '/crawl/guide']) site.remove(p); + } + }); + + it('skips subsequent pages when same-origin link redirects outside crawl origin', async () => { + const external = await fixtureSite(); + site.set('/crawl/out/start', 'LeaveStay'); + site.set('/crawl/out/leave', '', 'text/plain', 302, { location: `${external.url}/external-secret` }); + site.set('/crawl/out/stay', '

Stayed

'); + external.set('/external-secret', '

Secret external content

'); + try { + const res = await discover([{ type: 'crawl', url: `${site.url}/crawl/out/start` }], discoverOptions()); + expect(res.pages.map((p) => p.url.replace(site.url, ''))).toEqual(['/crawl/out/start', '/crawl/out/stay']); + expect(res.pages.some((p) => p.html?.includes('Secret external content'))).toBe(false); + expect(res.errors).toEqual([]); + } finally { + await external.close(); + for (const p of ['/crawl/out/start', '/crawl/out/leave', '/crawl/out/stay']) site.remove(p); + } + }); + + it('handles body-read failures without aborting the crawl', async () => { + site.set('/crawl/fail/start', 'BrokenGood'); + site.set('/crawl/fail/broken', '

Placeholder

'); + site.set('/crawl/fail/good', '

Good content

'); + const failingUrl = `${site.url}/crawl/fail/broken`; + const wrappedFetch = async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url === failingUrl) { + const stream = new ReadableStream({ + start(controller) { + controller.error(new Error('failing-stream body read error')); + }, + }); + const res = new Response(stream, { status: 200, headers: { 'content-type': 'text/html' } }); + Object.defineProperty(res, 'url', { value: failingUrl }); + return res; + } + return fetch(input, init); + }; + + try { + const res = await discover([{ type: 'crawl', url: `${site.url}/crawl/fail/start` }], { + ...discoverOptions(), + fetch: wrappedFetch as typeof fetch, + }); + expect(res.pages.map((p) => p.url.replace(site.url, ''))).toEqual(['/crawl/fail/start', '/crawl/fail/good']); + expect(res.errors).toContainEqual(expect.stringContaining('failing-stream body read error')); + } finally { + for (const p of ['/crawl/fail/start', '/crawl/fail/broken', '/crawl/fail/good']) site.remove(p); + } + }); + + it('reports crawl startup failures for include/exclude, robots disallow, and non-HTML responses', async () => { + site.set('/crawl/startfail/excluded', '

Excluded

'); + const excludedRes = await discover([{ type: 'crawl', url: `${site.url}/crawl/startfail/excluded` }], { + ...discoverOptions(), + exclude: ['**/excluded'], + }); + expect(excludedRes.pages).toEqual([]); + expect(excludedRes.errors).toContainEqual(expect.stringMatching(/does not match include\/exclude rules/)); + + site.set('/robots.txt', 'User-agent: *\nDisallow: /crawl/startfail/blocked', 'text/plain'); + site.set('/crawl/startfail/blocked', '

Blocked

'); + try { + const blockedRes = await discover([{ type: 'crawl', url: `${site.url}/crawl/startfail/blocked` }], discoverOptions()); + expect(blockedRes.pages).toEqual([]); + expect(blockedRes.errors).toContainEqual(expect.stringMatching(/disallowed by robots\.txt/)); + } finally { + site.remove('/robots.txt'); + } + + site.set('/crawl/startfail/file.pdf', '%PDF-1.4', 'application/pdf'); + const nonHtmlRes = await discover([{ type: 'crawl', url: `${site.url}/crawl/startfail/file.pdf` }], discoverOptions()); + expect(nonHtmlRes.pages).toEqual([]); + expect(nonHtmlRes.errors).toContainEqual(expect.stringMatching(/not HTML \(application\/pdf\)/)); + + for (const p of ['/crawl/startfail/excluded', '/crawl/startfail/blocked', '/crawl/startfail/file.pdf']) site.remove(p); + }); + + it('preserves deterministic BFS ordering under concurrent responses regardless of network timing', async () => { + site.set('/crawl/det/start', 'AB'); + site.set('/crawl/det/a', 'A1A2'); + site.set('/crawl/det/b', 'B1B2'); + site.set('/crawl/det/a1', '

A1

'); + site.set('/crawl/det/a2', '

A2

'); + site.set('/crawl/det/b1', '

B1

'); + site.set('/crawl/det/b2', '

B2

'); + + const fetchWithInvertedTiming = async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith('/crawl/det/a')) { + await new Promise((r) => setTimeout(r, 50)); + } else if (url.endsWith('/crawl/det/b')) { + await new Promise((r) => setTimeout(r, 5)); + } + return fetch(input, init); + }; + + try { + const res = await discover([{ type: 'crawl', url: `${site.url}/crawl/det/start` }], { + ...discoverOptions(), + concurrency: 4, + fetch: fetchWithInvertedTiming as typeof fetch, + }); + expect(res.pages.map((p) => p.url.replace(site.url, ''))).toEqual([ + '/crawl/det/start', + '/crawl/det/a', + '/crawl/det/b', + '/crawl/det/a1', + '/crawl/det/a2', + '/crawl/det/b1', + '/crawl/det/b2', + ]); + expect(res.errors).toEqual([]); + } finally { + for (const p of ['/crawl/det/start', '/crawl/det/a', '/crawl/det/b', '/crawl/det/a1', '/crawl/det/a2', '/crawl/det/b1', '/crawl/det/b2']) { + site.remove(p); + } + } + }); + it('turns globs into anchored patterns', () => { expect(globToRegExp('https://acme.test/docs/*').test('https://acme.test/docs/a')).toBe(true); expect(globToRegExp('https://acme.test/docs/*').test('https://acme.test/docs/a/b')).toBe(false); @@ -465,4 +643,39 @@ describe('ingest', () => { for (const p of ['/crawl/ingest/start', '/crawl/ingest/a', '/crawl/ingest/b']) site.remove(p); } }); + + it('protects existing indexed pages from prune when discovery is partial or encounters startup errors', async () => { + const pageHtml = (title: string, body: string) => + `${title}

${title}

${body}

`; + site.set('/crawl/prune/start', pageHtml('Start', 'P1P2')); + site.set('/crawl/prune/p1', pageHtml('P1', 'Content P1')); + site.set('/crawl/prune/p2', pageHtml('P2', 'Content P2')); + + const { config } = setup({ + sources: [{ type: 'crawl', url: `${site.url}/crawl/prune/start` }], + prune: true, + }); + + try { + const initial = await ingest(config); + expect(initial.indexed).toBe(3); + expect(initial.pruned).toBe(0); + + // Re-run with maxPages: 2, prune enabled -> partial discovery must protect p2 + const partial = await ingest({ ...config, maxPages: 2 }); + expect(partial.errors).toContain('stopped at --max-pages 2'); + expect(partial.pruned).toBe(0); + expect(partial.pruneSkipped).toBeDefined(); + + // Re-run with startup failure (e.g. start page disallowed by robots) -> must protect all indexed pages + site.set('/robots.txt', 'User-agent: *\nDisallow: /crawl/prune/start', 'text/plain'); + const startupFail = await ingest(config); + expect(startupFail.errors.length).toBeGreaterThan(0); + expect(startupFail.pruned).toBe(0); + expect(startupFail.pruneSkipped).toBeDefined(); + } finally { + site.remove('/robots.txt'); + for (const p of ['/crawl/prune/start', '/crawl/prune/p1', '/crawl/prune/p2']) site.remove(p); + } + }); }); diff --git a/website/src/content/docs/core/knowledge-base.mdx b/website/src/content/docs/core/knowledge-base.mdx index b3f0fc6b..d4db9cc3 100644 --- a/website/src/content/docs/core/knowledge-base.mdx +++ b/website/src/content/docs/core/knowledge-base.mdx @@ -144,7 +144,7 @@ Network sources respect `robots.txt` by default, using the `ChatCopsBot` group o | `{ type: 'sitemap', url }` | A sitemap, a sitemap index, or a gzipped one (`.xml.gz`) | | `{ type: 'urls', urls }` | The pages you list | | `{ type: 'directory', path, baseUrl }` | Local `.md`, `.mdx` and `.html` files, cited as if published under `baseUrl` | -| `{ type: 'crawl', url, maxDepth?, maxPages? }` | Crawl a site breadth-first from `url`, following same-origin links (defaults: `maxDepth: 3`, `maxPages: 500`) | +| `{ type: 'crawl', url, maxDepth?, maxPages? }` | Crawl a site breadth-first from `url`, following same-origin links (defaults: `maxDepth: 3`, `maxPages: 500`). `maxDepth` counts link hops from the starting URL; depth 0 means only the starting URL. `include` and `exclude` also control which crawl links are followed, including the starting URL. For example, `include: ['https://example.com/docs/**']` with `url: 'https://example.com/'` discovers nothing because the starting URL does not match `include`. | The rest of the config: From 08cc7e49e5030ce1f3145d7ae7e460c8ed8e1e2a Mon Sep 17 00:00:00 2001 From: bhushan4work Date: Fri, 6 Nov 2026 15:55:10 +0530 Subject: [PATCH 3/3] fix(ingest): robots check on start URL, source maxPages message, and doc (#586) --- packages/ingest/src/discover.ts | 35 +++++++++++++++++------ packages/ingest/tests/ingest.test.ts | 42 +++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/packages/ingest/src/discover.ts b/packages/ingest/src/discover.ts index 05a89f07..742a7e29 100644 --- a/packages/ingest/src/discover.ts +++ b/packages/ingest/src/discover.ts @@ -18,7 +18,7 @@ export interface DiscoverOptions { concurrency?: number; /** Obey robots.txt during crawl. Default true. */ respectRobots?: boolean; - /** Limit total pages discovered. */ + /** Caps the number of pages discovered per crawl source, not sitemap or URL-list sources. */ maxPages?: number; } @@ -141,12 +141,16 @@ async function crawlSite( errors: string[] ): Promise { const maxDepth = source.maxDepth ?? DEFAULT_CRAWL_DEPTH; - let effectiveMaxPages = source.maxPages ?? DEFAULT_CRAWL_PAGES; - if (options.maxPages !== undefined && options.maxPages >= 0) { - effectiveMaxPages = Math.min(effectiveMaxPages, Math.floor(options.maxPages)); - } + const sourceLimit = source.maxPages ?? DEFAULT_CRAWL_PAGES; + const globalLimit = options.maxPages !== undefined && options.maxPages >= 0 ? Math.floor(options.maxPages) : undefined; + const isGlobalLimit = globalLimit !== undefined && globalLimit <= sourceLimit; + const effectiveMaxPages = isGlobalLimit ? globalLimit : sourceLimit; + const capMessage = isGlobalLimit + ? `stopped at --max-pages ${effectiveMaxPages}` + : `${source.url}: crawl stopped at maxPages ${effectiveMaxPages}`; + if (effectiveMaxPages <= 0) { - errors.push(`stopped at --max-pages ${effectiveMaxPages}`); + errors.push(capMessage); return; } @@ -189,6 +193,19 @@ async function crawlSite( return pending; } + if (respectRobots) { + let startRobots: Robots | undefined; + try { + startRobots = await getRobots(startUrl); + } catch { + startRobots = undefined; + } + if (startRobots && !startRobots.allowed(startUrl)) { + errors.push(`${startUrl}: disallowed by robots.txt`); + return; + } + } + let startResponse: Response; try { startResponse = await options.fetch(startUrl, { @@ -268,7 +285,7 @@ async function crawlSite( if (count >= effectiveMaxPages) { if (currentLevel.length > 0) { - errors.push(`stopped at --max-pages ${effectiveMaxPages}`); + errors.push(capMessage); } return; } @@ -412,8 +429,8 @@ async function crawlSite( if (count >= effectiveMaxPages) { if (nextIndex < previousLength || currentLevel.length > 0) { - if (!errors.includes(`stopped at --max-pages ${effectiveMaxPages}`)) { - errors.push(`stopped at --max-pages ${effectiveMaxPages}`); + if (!errors.includes(capMessage)) { + errors.push(capMessage); } } break; diff --git a/packages/ingest/tests/ingest.test.ts b/packages/ingest/tests/ingest.test.ts index de461b0d..38c5a1f3 100644 --- a/packages/ingest/tests/ingest.test.ts +++ b/packages/ingest/tests/ingest.test.ts @@ -113,12 +113,37 @@ describe('discover', () => { expect(res.pages).toHaveLength(2); expect(res.pages[0].url).toBe(`${site.url}/crawl/mp/start`); expect(res.pages[1].url).toBe(`${site.url}/crawl/mp/a`); - expect(res.errors).toEqual(['stopped at --max-pages 2']); + expect(res.errors).toEqual([`${site.url}/crawl/mp/start: crawl stopped at maxPages 2`]); const requests = site.requests.slice(before); expect(requests).toContain('/crawl/mp/start'); expect(requests).toContain('/crawl/mp/a'); expect(requests).not.toContain('/crawl/mp/b'); expect(requests).not.toContain('/crawl/mp/c'); + + const globalRes = await discover([{ type: 'crawl', url: `${site.url}/crawl/mp/start` }], { + ...discoverOptions(), + maxPages: 2, + }); + expect(globalRes.pages).toHaveLength(2); + expect(globalRes.pages[0].url).toBe(`${site.url}/crawl/mp/start`); + expect(globalRes.pages[1].url).toBe(`${site.url}/crawl/mp/a`); + expect(globalRes.errors).toEqual(['stopped at --max-pages 2']); + + // Source limit responsible when lower than global limit + const sourceWins = await discover( + [{ type: 'crawl', url: `${site.url}/crawl/mp/start`, maxPages: 2 }], + { ...discoverOptions(), maxPages: 3 } + ); + expect(sourceWins.pages).toHaveLength(2); + expect(sourceWins.errors).toEqual([`${site.url}/crawl/mp/start: crawl stopped at maxPages 2`]); + + // Global limit responsible when lower than source limit + const globalWins = await discover( + [{ type: 'crawl', url: `${site.url}/crawl/mp/start`, maxPages: 3 }], + { ...discoverOptions(), maxPages: 2 } + ); + expect(globalWins.pages).toHaveLength(2); + expect(globalWins.errors).toEqual(['stopped at --max-pages 2']); } finally { for (const p of ['/crawl/mp/start', '/crawl/mp/a', '/crawl/mp/b', '/crawl/mp/c']) site.remove(p); } @@ -311,6 +336,7 @@ describe('discover', () => { const blockedRes = await discover([{ type: 'crawl', url: `${site.url}/crawl/startfail/blocked` }], discoverOptions()); expect(blockedRes.pages).toEqual([]); expect(blockedRes.errors).toContainEqual(expect.stringMatching(/disallowed by robots\.txt/)); + expect(site.requests).not.toContain('/crawl/startfail/blocked'); } finally { site.remove('/robots.txt'); } @@ -323,6 +349,20 @@ describe('discover', () => { for (const p of ['/crawl/startfail/excluded', '/crawl/startfail/blocked', '/crawl/startfail/file.pdf']) site.remove(p); }); + it('disallows starting page when redirect destination is disallowed by robots.txt', async () => { + site.set('/robots.txt', 'User-agent: *\nDisallow: /crawl/redir/blocked', 'text/plain'); + site.set('/crawl/redir/allowed-start', '', 'text/plain', 302, { location: `${site.url}/crawl/redir/blocked` }); + site.set('/crawl/redir/blocked', '

Blocked landing

'); + try { + const res = await discover([{ type: 'crawl', url: `${site.url}/crawl/redir/allowed-start` }], discoverOptions()); + expect(res.pages).toEqual([]); + expect(res.errors).toContain(`${site.url}/crawl/redir/blocked: disallowed by robots.txt`); + } finally { + site.remove('/robots.txt'); + for (const p of ['/crawl/redir/allowed-start', '/crawl/redir/blocked']) site.remove(p); + } + }); + it('preserves deterministic BFS ordering under concurrent responses regardless of network timing', async () => { site.set('/crawl/det/start', 'AB'); site.set('/crawl/det/a', 'A1A2');