From 45565e276d5511be4232c0b2c9c667db26c6a54e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Tue, 25 Aug 2026 09:07:14 +0200 Subject: [PATCH] feat: add `initialConcurrency` crawler option (#4067) Exposes the default `ConcurrencySystem`'s `desiredConcurrency` as an `initialConcurrency` crawler option, alongside the existing min/max shortcuts. Closes #4066 --- docs/public-api/crawlee-basic.api.md | 1 + .../src/internals/basic-crawler.ts | 29 ++++++++++++++----- test/core/crawlers/basic_crawler.test.ts | 25 +++++++++++++--- test/core/crawlers/http_crawler.test.ts | 12 ++++++++ 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md index 466ff3cac52b..9b543bd6752d 100644 --- a/docs/public-api/crawlee-basic.api.md +++ b/docs/public-api/crawlee-basic.api.md @@ -124,6 +124,7 @@ export interface BasicCrawlerOptions Number.isInteger(value) || value === Infinity, 'Expected an integer or infinite number') .refine((value) => value >= 1, 'Expected a number greater than or equal to 1') @@ -987,6 +995,7 @@ export class BasicCrawler< // AutoscaledPool shorthands minConcurrency, maxConcurrency, + initialConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, @@ -1011,12 +1020,15 @@ export class BasicCrawler< // hammering a site. if ( concurrencySystem !== undefined && - (minConcurrency !== undefined || maxConcurrency !== undefined || maxRequestsPerMinute !== undefined) + (minConcurrency !== undefined || + maxConcurrency !== undefined || + initialConcurrency !== undefined || + maxRequestsPerMinute !== undefined) ) { throw new Error( - 'The `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts cannot be combined with ' + - '`concurrencySystem` - they configure the default `ConcurrencySystem` that a supplied one ' + - 'replaces. Pass them to the `ConcurrencySystem` constructor instead.', + 'The `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts ' + + 'cannot be combined with `concurrencySystem` - they configure the default `ConcurrencySystem` ' + + 'that a supplied one replaces. Pass them to the `ConcurrencySystem` constructor instead.', ); } @@ -1348,6 +1360,9 @@ export class BasicCrawler< minConcurrency, maxConcurrency, maxTasksPerMinute: maxRequestsPerMinute, + // Spread conditionally - an explicit `undefined` would clobber a subclass default, see + // `HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS`. + ...(initialConcurrency !== undefined && { desiredConcurrency: initialConcurrency }), log: this.log, }), ); @@ -1358,7 +1373,7 @@ export class BasicCrawler< /** * Builds the crawler-owned default {@apilink ConcurrencySystem} from the resolved - * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a + * `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a * {@apilink BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected. * * Subclasses may override this to tune the default system (e.g. {@apilink HttpCrawler} raises the starting diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts index 0dac5e1d5a8b..8cb7190b2152 100644 --- a/test/core/crawlers/basic_crawler.test.ts +++ b/test/core/crawlers/basic_crawler.test.ts @@ -276,7 +276,7 @@ describe('BasicCrawler', () => { expect(bookedFor).toEqual(new Set(['crawler-a', 'crawler-b'])); }); - test.each(['minConcurrency', 'maxConcurrency', 'maxRequestsPerMinute'] as const)( + test.each(['minConcurrency', 'maxConcurrency', 'initialConcurrency', 'maxRequestsPerMinute'] as const)( 'throws when %s is combined with a supplied concurrencySystem', (shortcut) => { expect( @@ -624,6 +624,7 @@ describe('BasicCrawler', () => { const collect = (crawler: BasicCrawler) => ({ minConcurrency: (crawler.concurrencySystem! as ConcurrencySystem).minConcurrency, maxConcurrency: (crawler.concurrencySystem! as ConcurrencySystem).maxConcurrency, + desiredConcurrency: (crawler.concurrencySystem! as ConcurrencySystem).desiredConcurrency, // eslint-disable-next-line dot-notation -- private member on the governor maxTasksPerMinute: (crawler.concurrencySystem! as ConcurrencySystem)['maxTasksPerMinute'], }); @@ -634,11 +635,17 @@ describe('BasicCrawler', () => { requestHandler, minConcurrency: 123, maxConcurrency: 456, + initialConcurrency: 234, maxRequestsPerMinute: 789, }); // An injected system carries its own config (the shortcuts are rejected alongside one, see above). - const injectedSystem = new ConcurrencySystem({ minConcurrency: 16, maxConcurrency: 32, maxTasksPerMinute: 64 }); + const injectedSystem = new ConcurrencySystem({ + minConcurrency: 16, + maxConcurrency: 32, + desiredConcurrency: 24, + maxTasksPerMinute: 64, + }); const injected = new BasicCrawler({ requestList, requestHandler, @@ -650,8 +657,18 @@ describe('BasicCrawler', () => { await Promise.all([shortcuts.run(), injected.run()]); await injectedSystem.stop(); - expect(collect(shortcuts)).toEqual({ minConcurrency: 123, maxConcurrency: 456, maxTasksPerMinute: 789 }); - expect(collect(injected)).toEqual({ minConcurrency: 16, maxConcurrency: 32, maxTasksPerMinute: 64 }); + expect(collect(shortcuts)).toEqual({ + minConcurrency: 123, + maxConcurrency: 456, + desiredConcurrency: 234, + maxTasksPerMinute: 789, + }); + expect(collect(injected)).toEqual({ + minConcurrency: 16, + maxConcurrency: 32, + desiredConcurrency: 24, + maxTasksPerMinute: 64, + }); // The injected system is the very instance the pool uses. expect(injected.concurrencySystem!).toBe(injectedSystem); }); diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.ts index 1ba725d90efa..d18cae419982 100644 --- a/test/core/crawlers/http_crawler.test.ts +++ b/test/core/crawlers/http_crawler.test.ts @@ -193,6 +193,18 @@ test('concurrency shortcuts coexist with the HTTP-optimized defaults', async () expect(crawler.asConfigured!.desiredConcurrency).toBe(5); }); +test('initialConcurrency overrides the HTTP-optimized starting concurrency', async () => { + const crawler = new ObservableHttpCrawler({ + initialConcurrency: 3, + maxRequestRetries: 0, + requestHandler: () => {}, + }); + + await crawler.run([url]); + + expect(crawler.asConfigured!.desiredConcurrency).toBe(3); +}); + test('parseWithCheerio works', async () => { const results: string[] = [];