feat(ingest): crawl discovery source (#586)

.changeset/crawl-discovery.md

@@ -0,0 +1,6 @@
+---
+'@chatcops/ingest': minor
+'@chatcops/cli': minor
+---
+
+Support crawl ingestion sources starting from a URL and following same-origin links breadth-first with maxDepth, maxPages, deduplication, robots.txt compliance, and cached HTML reuse during extraction. Add --crawl to the chatcops ingest command and crawl: URLs to --source.

api/ingest.api.md

@@ -27,6 +27,7 @@ export declare function discover(sources: IngestSource[], options: DiscoverOptio
 export interface DiscoveredPage {
     url: string;
     file?: string;
+    html?: string;
 }
 
 export interface DiscoverOptions {
@@ -35,6 +36,9 @@ export interface DiscoverOptions {
     timeoutMs: number;
     include?: string[];
     exclude?: string[];
+    concurrency?: number;
+    respectRobots?: boolean;
+    maxPages?: number;
 }
 
 export interface Discovery {
@@ -114,6 +118,11 @@ export type IngestSource = {
     baseUrl: string;
     path: string;
     type: 'directory';
+} | {
+    maxDepth?: number;
+    maxPages?: number;
+    type: 'crawl';
+    url: string;
 };
 
 export interface IngestStats {

packages/cli/README.md

@@ -25,7 +25,7 @@ export default defineConfig({
 });
 ```
 
-`chatcops ingest` options: `--source` (repeatable; a sitemap, a page URL, or a folder with `--base-url`), `--index`, `--dry-run`, `--max-pages`, `--concurrency`, `--prune`, `--force`, `--json`. `chatcops search` options: `--index`, `--top-k`, `--json`. Run `chatcops --help` for details.
+`chatcops ingest` options: `--source` (repeatable; a sitemap, a page URL, or a folder with `--base-url`), `--crawl`, `--index`, `--dry-run`, `--max-pages`, `--concurrency`, `--prune`, `--force`, `--json`. `chatcops search` options: `--index`, `--top-k`, `--json`. Run `chatcops --help` for details.
 
 `chatcops test` runs eval files (`evals/**/*.chat.yaml` by default) against the bot in the config's `server` section:
 

packages/cli/src/main.ts

@@ -34,8 +34,9 @@ Usage:
   chatcops test [files...] [options]        Run eval files against your bot
 
 Ingest options:
-  --source <url|path>   A sitemap (.xml, .xml.gz), a page URL, or a folder of .md/.mdx/.html
-                        files (with --base-url). Repeatable. Replaces the config's sources.
+  --source <url|path>   A sitemap (.xml, .xml.gz), crawl:<url>, a page URL, or a folder of
+                        .md/.mdx/.html files (with --base-url). Repeatable. Replaces the config's sources.
+  --crawl <url>         Crawl same-origin pages starting at url. Replaces the config's sources.
   --base-url <url>      Where a folder source is published
   --index <name>        Index name (default "default")
   --dry-run             Discover, extract and chunk; write and embed nothing
@@ -125,6 +126,7 @@ function parse(argv: string[]) {
     options: {
       config: { type: 'string' },
       source: { type: 'string', multiple: true },
+      crawl: { type: 'string' },
       'base-url': { type: 'string' },
       index: { type: 'string' },
       'dry-run': { type: 'boolean' },
@@ -158,8 +160,11 @@ async function runIngest(values: Values, io: Io): Promise<number> {
   const loaded = await loadConfig(io.cwd, values.config);
   try {
     const target = indexConfig(loaded);
-    const sources = values.source?.length ? values.source.map((source) => toSource(source, values['base-url'], io.cwd)) : loaded.config.ingest?.sources;
-    if (!sources?.length) throw new ConfigError('Nothing to ingest: pass --source, or set ingest.sources in the config.');
+    const cliSources: IngestSource[] = [];
+    if (values.crawl) cliSources.push({ type: 'crawl', url: values.crawl });
+    if (values.source?.length) cliSources.push(...values.source.map((source) => toSource(source, values['base-url'], io.cwd)));
+    const sources = cliSources.length ? cliSources : loaded.config.ingest?.sources;
+    if (!sources?.length) throw new ConfigError('Nothing to ingest: pass --source, --crawl, or set ingest.sources in the config.');
 
     // A model only runs with --contextualize, so a run never spends on one by surprise: the config holds which.
     if (values.contextualize && !loaded.config.ingest?.contextualize) {
@@ -221,6 +226,9 @@ async function runSearch(question: string, values: Values, io: Io): Promise<numb
 
 /** A `--source` value: a sitemap URL, a page URL, or a folder (with `--base-url`). */
 export function toSource(value: string, baseUrl: string | undefined, cwd: string): IngestSource {
+  if (/^crawl:https?:\/\//i.test(value)) {
+    return { type: 'crawl', url: value.slice('crawl:'.length) };
+  }
   if (/^https?:\/\//i.test(value)) {
     return /\.xml(\.gz)?(\?.*)?$/i.test(value) ? { type: 'sitemap', url: value } : { type: 'urls', urls: [value] };
   }

packages/cli/tests/cli.test.ts

@@ -250,10 +250,44 @@ export default {
     const cwd = project({ shared: 'folder' });
     expect(await run(['ingest', '--source', './docs'], cwd)).toMatchObject({ code: EXIT.unusable, stderr: expect.stringContaining('needs --base-url') });
   });
+
+  it('errors when nothing to ingest without source or crawl flags', async () => {
+    const cwd = project({
+      shared: 'no-sources',
+      body: `import { memoryStorage, memoryVectors } from '@chatcops/core';
+export default {
+  storage: { ...memoryStorage(), vectors: memoryVectors({ dimensions: 4 }) },
+  embedder: { id: 'test:fake@4', dimensions: 4, async embed() { return []; } },
+  ingest: {},
+};`,
+    });
+    expect(await run(['ingest'], cwd)).toMatchObject({
+      code: EXIT.unusable,
+      stderr: expect.stringContaining('Nothing to ingest: pass --source, --crawl, or set ingest.sources in the config.'),
+    });
+  });
+
+  it('ingests with --crawl and crawl: source', async () => {
+    site.set('/crawl/cli/start', '<!doctype html><html><body><main><h1>Start</h1><p>Start with <a href="/crawl/cli/sub">Sub</a></p></main></body></html>');
+    site.set('/crawl/cli/sub', '<!doctype html><html><body><main><h1>Sub</h1><p>Sub content</p></main></body></html>');
+    const cwd = project({ shared: 'cli-crawl' });
+    try {
+      const crawlFlag = await run(['ingest', '--crawl', `${site.url}/crawl/cli/start`], cwd);
+      expect(crawlFlag.code).toBe(EXIT.ok);
+      expect(crawlFlag.stdout).toContain('Indexed 2 of 2 pages');
+
+      const crawlSource = await run(['ingest', '--source', `crawl:${site.url}/crawl/cli/start`, '--force'], cwd);
+      expect(crawlSource.code).toBe(EXIT.ok);
+      expect(crawlSource.stdout).toContain('Indexed 2 of 2 pages');
+    } finally {
+      for (const p of ['/crawl/cli/start', '/crawl/cli/sub']) site.remove(p);
+    }
+  });
 });
 
 describe('toSource', () => {
   it('tells sitemaps, pages and folders apart', () => {
+    expect(toSource('crawl:https://acme.test/start', undefined, '/x')).toEqual({ type: 'crawl', url: 'https://acme.test/start' });
     expect(toSource('https://acme.test/sitemap.xml', undefined, '/x')).toEqual({ type: 'sitemap', url: 'https://acme.test/sitemap.xml' });
     expect(toSource('https://acme.test/sitemap.xml.gz?v=2', undefined, '/x')).toEqual({ type: 'sitemap', url: 'https://acme.test/sitemap.xml.gz?v=2' });
     expect(toSource('https://acme.test/pricing', undefined, '/x')).toEqual({ type: 'urls', urls: ['https://acme.test/pricing'] });