feat(api): manual updates

Author: stainless-app[bot]

.stats.yml

@@ -1,4 +1,4 @@
-configured_endpoints: 20
+configured_endpoints: 21
 openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev%2Fcontext.dev-97cdb78dc0d72e9df643a89660f2b0c9687f12c6e4d93f7767f6cfc1b4f2e4c7.yml
 openapi_spec_hash: 92fc94fd8865fabe78c2667490ca3884
-config_hash: 51eb368cba05800d9497df4fa318828e
+config_hash: 682b89b02a20f5d1c13e2c91ecbcf5ce

api.md

@@ -3,6 +3,7 @@
 Types:
 
 - <code><a href="./src/resources/web.ts">WebScreenshotResponse</a></code>
+- <code><a href="./src/resources/web.ts">WebWebCrawlMdResponse</a></code>
 - <code><a href="./src/resources/web.ts">WebWebScrapeHTMLResponse</a></code>
 - <code><a href="./src/resources/web.ts">WebWebScrapeImagesResponse</a></code>
 - <code><a href="./src/resources/web.ts">WebWebScrapeMdResponse</a></code>
@@ -11,6 +12,7 @@ Types:
 Methods:
 
 - <code title="get /brand/screenshot">client.web.<a href="./src/resources/web.ts">screenshot</a>({ ...params }) -> WebScreenshotResponse</code>
+- <code title="post /web/crawl">client.web.<a href="./src/resources/web.ts">webCrawlMd</a>({ ...params }) -> WebWebCrawlMdResponse</code>
 - <code title="get /web/scrape/html">client.web.<a href="./src/resources/web.ts">webScrapeHTML</a>({ ...params }) -> WebWebScrapeHTMLResponse</code>
 - <code title="get /web/scrape/images">client.web.<a href="./src/resources/web.ts">webScrapeImages</a>({ ...params }) -> WebWebScrapeImagesResponse</code>
 - <code title="get /web/scrape/markdown">client.web.<a href="./src/resources/web.ts">webScrapeMd</a>({ ...params }) -> WebWebScrapeMdResponse</code>

packages/mcp-server/src/code-tool-worker.ts

@@ -109,6 +109,7 @@ function getTSDiagnostics(code: string): string[] {
 const fuse = new Fuse(
   [
     'client.web.screenshot',
+    'client.web.webCrawlMd',
     'client.web.webScrapeHTML',
     'client.web.webScrapeImages',
     'client.web.webScrapeMd',

packages/mcp-server/src/local-docs-search.ts

@@ -237,6 +237,52 @@ const EMBEDDED_METHODS: MethodEntry[] = [
       },
     },
   },
+  {
+    name: 'web_crawl_md',
+    endpoint: '/web/crawl',
+    httpMethod: 'post',
+    summary: 'Crawl website and extract Markdown',
+    description:
+      'Performs a crawl starting from a given URL, extracts page content as Markdown, and returns results for all crawled pages. Only follows links within the same domain as the starting URL. Costs 1 credit per successful page crawled.',
+    stainlessPath: '(resource) web > (method) web_crawl_md',
+    qualified: 'client.web.webCrawlMd',
+    params: [
+      'url: string;',
+      'followSubdomains?: boolean;',
+      'includeImages?: boolean;',
+      'includeLinks?: boolean;',
+      'maxDepth?: number;',
+      'maxPages?: number;',
+      'shortenBase64Images?: boolean;',
+      'urlRegex?: string;',
+      'useMainContentOnly?: boolean;',
+    ],
+    response:
+      '{ metadata: { maxCrawlDepth: number; numFailed: number; numSucceeded: number; numUrls: number; }; results: { markdown: string; metadata: { crawlDepth: number; statusCode: number; success: boolean; title: string; url: string; }; }[]; }',
+    markdown:
+      "## web_crawl_md\n\n`client.web.webCrawlMd(url: string, followSubdomains?: boolean, includeImages?: boolean, includeLinks?: boolean, maxDepth?: number, maxPages?: number, shortenBase64Images?: boolean, urlRegex?: string, useMainContentOnly?: boolean): { metadata: object; results: object[]; }`\n\n**post** `/web/crawl`\n\nPerforms a crawl starting from a given URL, extracts page content as Markdown, and returns results for all crawled pages. Only follows links within the same domain as the starting URL. Costs 1 credit per successful page crawled.\n\n### Parameters\n\n- `url: string`\n  The starting URL for the crawl (must include http:// or https:// protocol)\n\n- `followSubdomains?: boolean`\n  When true, follow links on subdomains of the starting URL's domain (e.g. docs.example.com when starting from example.com). www and apex are always treated as equivalent.\n\n- `includeImages?: boolean`\n  Include image references in the Markdown output\n\n- `includeLinks?: boolean`\n  Preserve hyperlinks in the Markdown output\n\n- `maxDepth?: number`\n  Maximum link depth from the starting URL (0 = only the starting page)\n\n- `maxPages?: number`\n  Maximum number of pages to crawl. Hard cap: 500.\n\n- `shortenBase64Images?: boolean`\n  Truncate base64-encoded image data in the Markdown output\n\n- `urlRegex?: string`\n  Regex pattern. Only URLs matching this pattern will be followed and scraped.\n\n- `useMainContentOnly?: boolean`\n  Extract only the main content, stripping headers, footers, sidebars, and navigation\n\n### Returns\n\n- `{ metadata: { maxCrawlDepth: number; numFailed: number; numSucceeded: number; numUrls: number; }; results: { markdown: string; metadata: { crawlDepth: number; statusCode: number; success: boolean; title: string; url: string; }; }[]; }`\n\n  - `metadata: { maxCrawlDepth: number; numFailed: number; numSucceeded: number; numUrls: number; }`\n  - `results: { markdown: string; metadata: { crawlDepth: number; statusCode: number; success: boolean; title: string; url: string; }; }[]`\n\n### Example\n\n```typescript\nimport ContextDev from 'context.dev';\n\nconst client = new ContextDev();\n\nconst response = await client.web.webCrawlMd({ url: 'https://example.com' });\n\nconsole.log(response);\n```",
+    perLanguage: {
+      http: {
+        example:
+          'curl https://api.context.dev/v1/web/crawl \\\n    -H \'Content-Type: application/json\' \\\n    -H "Authorization: Bearer $CONTEXT_DEV_API_KEY" \\\n    -d \'{\n          "url": "https://example.com"\n        }\'',
+      },
+      python: {
+        method: 'web.web_crawl_md',
+        example:
+          'import os\nfrom context.dev import ContextDev\n\nclient = ContextDev(\n    api_key=os.environ.get("CONTEXT_DEV_API_KEY"),  # This is the default and can be omitted\n)\nresponse = client.web.web_crawl_md(\n    url="https://example.com",\n)\nprint(response.metadata)',
+      },
+      ruby: {
+        method: 'web.web_crawl_md',
+        example:
+          'require "context_dev"\n\ncontext_dev = ContextDev::Client.new(api_key: "My API Key")\n\nresponse = context_dev.web.web_crawl_md(url: "https://example.com")\n\nputs(response)',
+      },
+      typescript: {
+        method: 'client.web.webCrawlMd',
+        example:
+          "import ContextDev from 'context.dev';\n\nconst client = new ContextDev({\n  apiKey: process.env['CONTEXT_DEV_API_KEY'], // This is the default and can be omitted\n});\n\nconst response = await client.web.webCrawlMd({ url: 'https://example.com' });\n\nconsole.log(response.metadata);",
+      },
+    },
+  },
   {
     name: 'extract_products',
     endpoint: '/brand/ai/products',

packages/mcp-server/src/methods.ts

@@ -16,6 +16,12 @@ export const sdkMethods: SdkMethod[] = [
     httpMethod: 'get',
     httpPath: '/brand/screenshot',
   },
+  {
+    clientCallName: 'client.web.webCrawlMd',
+    fullyQualifiedName: 'web.webCrawlMd',
+    httpMethod: 'post',
+    httpPath: '/web/crawl',
+  },
   {
     clientCallName: 'client.web.webScrapeHTML',
     fullyQualifiedName: 'web.webScrapeHTML',

src/client.ts

@@ -62,6 +62,8 @@ import {
   Web,
   WebScreenshotParams,
   WebScreenshotResponse,
+  WebWebCrawlMdParams,
+  WebWebCrawlMdResponse,
   WebWebScrapeHTMLParams,
   WebWebScrapeHTMLResponse,
   WebWebScrapeImagesParams,
@@ -792,11 +794,13 @@ export declare namespace ContextDev {
   export {
     Web as Web,
     type WebScreenshotResponse as WebScreenshotResponse,
+    type WebWebCrawlMdResponse as WebWebCrawlMdResponse,
     type WebWebScrapeHTMLResponse as WebWebScrapeHTMLResponse,
     type WebWebScrapeImagesResponse as WebWebScrapeImagesResponse,
     type WebWebScrapeMdResponse as WebWebScrapeMdResponse,
     type WebWebScrapeSitemapResponse as WebWebScrapeSitemapResponse,
     type WebScreenshotParams as WebScreenshotParams,
+    type WebWebCrawlMdParams as WebWebCrawlMdParams,
     type WebWebScrapeHTMLParams as WebWebScrapeHTMLParams,
     type WebWebScrapeImagesParams as WebWebScrapeImagesParams,
     type WebWebScrapeMdParams as WebWebScrapeMdParams,

src/resources/index.ts

@@ -44,11 +44,13 @@ export {
 export {
   Web,
   type WebScreenshotResponse,
+  type WebWebCrawlMdResponse,
   type WebWebScrapeHTMLResponse,
   type WebWebScrapeImagesResponse,
   type WebWebScrapeMdResponse,
   type WebWebScrapeSitemapResponse,
   type WebScreenshotParams,
+  type WebWebCrawlMdParams,
   type WebWebScrapeHTMLParams,
   type WebWebScrapeImagesParams,
   type WebWebScrapeMdParams,

src/resources/web.ts

@@ -15,6 +15,15 @@ export class Web extends APIResource {
     return this._client.get('/brand/screenshot', { query, ...options });
   }
 
+  /**
+   * Performs a crawl starting from a given URL, extracts page content as Markdown,
+   * and returns results for all crawled pages. Only follows links within the same
+   * domain as the starting URL. Costs 1 credit per successful page crawled.
+   */
+  webCrawlMd(body: WebWebCrawlMdParams, options?: RequestOptions): APIPromise<WebWebCrawlMdResponse> {
+    return this._client.post('/web/crawl', { body, ...options });
+  }
+
   /**
    * Scrapes the given URL and returns the raw HTML content of the page.
    */
@@ -85,6 +94,74 @@ export interface WebScreenshotResponse {
   status?: string;
 }
 
+export interface WebWebCrawlMdResponse {
+  metadata: WebWebCrawlMdResponse.Metadata;
+
+  results: Array<WebWebCrawlMdResponse.Result>;
+}
+
+export namespace WebWebCrawlMdResponse {
+  export interface Metadata {
+    /**
+     * Maximum crawl depth reached during the crawl
+     */
+    maxCrawlDepth: number;
+
+    /**
+     * Number of pages that failed to crawl
+     */
+    numFailed: number;
+
+    /**
+     * Number of pages successfully crawled
+     */
+    numSucceeded: number;
+
+    /**
+     * Total number of URLs crawled
+     */
+    numUrls: number;
+  }
+
+  export interface Result {
+    /**
+     * Extracted page content as Markdown (empty string on failure)
+     */
+    markdown: string;
+
+    metadata: Result.Metadata;
+  }
+
+  export namespace Result {
+    export interface Metadata {
+      /**
+       * Depth relative to the start URL. 0 = start URL, 1 = one link away.
+       */
+      crawlDepth: number;
+
+      /**
+       * HTTP status code of the response
+       */
+      statusCode: number;
+
+      /**
+       * true if the page was fetched and parsed successfully
+       */
+      success: boolean;
+
+      /**
+       * The page's <title> content (empty string if unavailable)
+       */
+      title: string;
+
+      /**
+       * The URL that was fetched
+       */
+      url: string;
+    }
+  }
+}
+
 export interface WebWebScrapeHTMLResponse {
   /**
    * Raw HTML content of the page
@@ -239,6 +316,56 @@ export interface WebScreenshotParams {
   prioritize?: 'speed' | 'quality';
 }
 
+export interface WebWebCrawlMdParams {
+  /**
+   * The starting URL for the crawl (must include http:// or https:// protocol)
+   */
+  url: string;
+
+  /**
+   * When true, follow links on subdomains of the starting URL's domain (e.g.
+   * docs.example.com when starting from example.com). www and apex are always
+   * treated as equivalent.
+   */
+  followSubdomains?: boolean;
+
+  /**
+   * Include image references in the Markdown output
+   */
+  includeImages?: boolean;
+
+  /**
+   * Preserve hyperlinks in the Markdown output
+   */
+  includeLinks?: boolean;
+
+  /**
+   * Maximum link depth from the starting URL (0 = only the starting page)
+   */
+  maxDepth?: number;
+
+  /**
+   * Maximum number of pages to crawl. Hard cap: 500.
+   */
+  maxPages?: number;
+
+  /**
+   * Truncate base64-encoded image data in the Markdown output
+   */
+  shortenBase64Images?: boolean;
+
+  /**
+   * Regex pattern. Only URLs matching this pattern will be followed and scraped.
+   */
+  urlRegex?: string;
+
+  /**
+   * Extract only the main content, stripping headers, footers, sidebars, and
+   * navigation
+   */
+  useMainContentOnly?: boolean;
+}
+
 export interface WebWebScrapeHTMLParams {
   /**
    * Full URL to scrape (must include http:// or https:// protocol)
@@ -299,11 +426,13 @@ export interface WebWebScrapeSitemapParams {
 export declare namespace Web {
   export {
     type WebScreenshotResponse as WebScreenshotResponse,
+    type WebWebCrawlMdResponse as WebWebCrawlMdResponse,
     type WebWebScrapeHTMLResponse as WebWebScrapeHTMLResponse,
     type WebWebScrapeImagesResponse as WebWebScrapeImagesResponse,
     type WebWebScrapeMdResponse as WebWebScrapeMdResponse,
     type WebWebScrapeSitemapResponse as WebWebScrapeSitemapResponse,
     type WebScreenshotParams as WebScreenshotParams,
+    type WebWebCrawlMdParams as WebWebCrawlMdParams,
     type WebWebScrapeHTMLParams as WebWebScrapeHTMLParams,
     type WebWebScrapeImagesParams as WebWebScrapeImagesParams,
     type WebWebScrapeMdParams as WebWebScrapeMdParams,

tests/api-resources/web.test.ts

@@ -30,6 +30,33 @@ describe('resource web', () => {
     });
   });
 
+  // Mock server tests are disabled
+  test.skip('webCrawlMd: only required params', async () => {
+    const responsePromise = client.web.webCrawlMd({ url: 'https://example.com' });
+    const rawResponse = await responsePromise.asResponse();
+    expect(rawResponse).toBeInstanceOf(Response);
+    const response = await responsePromise;
+    expect(response).not.toBeInstanceOf(Response);
+    const dataAndResponse = await responsePromise.withResponse();
+    expect(dataAndResponse.data).toBe(response);
+    expect(dataAndResponse.response).toBe(rawResponse);
+  });
+
+  // Mock server tests are disabled
+  test.skip('webCrawlMd: required and optional params', async () => {
+    const response = await client.web.webCrawlMd({
+      url: 'https://example.com',
+      followSubdomains: true,
+      includeImages: true,
+      includeLinks: true,
+      maxDepth: 0,
+      maxPages: 1,
+      shortenBase64Images: true,
+      urlRegex: 'urlRegex',
+      useMainContentOnly: true,
+    });
+  });
+
   // Mock server tests are disabled
   test.skip('webScrapeHTML: only required params', async () => {
     const responsePromise = client.web.webScrapeHTML({ url: 'https://example.com' });