[pull] master from apify:master

docs/public-api/crawlee-basic.api.md

@@ -124,6 +124,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
     httpClient?: BaseHttpClient;
     id?: string;
     ignoreHttpErrorStatusCodes?: number[];
+    initialConcurrency?: number;
     keepAlive?: boolean;
     logger?: CrawleeLogger;
     maxConcurrency?: number;

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -392,8 +392,8 @@ export interface BasicCrawlerOptions<
      * single budget. Each crawler still builds and drives its own {@apilink AutoscaledPool}; only the load/scaling
      * accounting is shared.
      *
-     * Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts, which configure
-     * the default system this one replaces — combining the two throws.
+     * Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute`
+     * shortcuts, which configure the default system this one replaces — combining the two throws.
      *
      * You own a supplied system's lifecycle: `start()` it before `run()` (which throws otherwise) and `stop()` it once
      * every crawler borrowing it has finished. The crawler does neither on your behalf.
@@ -416,6 +416,13 @@ export interface BasicCrawlerOptions<
      */
     maxConcurrency?: number;
 
+    /**
+     * Sets the concurrency (parallelism) the crawl starts with, before any scaling happens. Shortcut for the
+     * {@apilink ConcurrencySystemOptions.desiredConcurrency|`desiredConcurrency`} option of the crawler's default
+     * {@apilink ConcurrencySystem}. Defaults to `minConcurrency`.
+     */
+    initialConcurrency?: number;
+
     /**
      * The maximum number of requests per minute the crawler should run.
      * By default, this is set to `Infinity`, but we can pass any positive, non-zero integer.
@@ -937,6 +944,7 @@ export class BasicCrawler<
         // AutoscaledPool shorthands
         minConcurrency: schemas.anyNumber.optional(),
         maxConcurrency: schemas.anyNumber.optional(),
+        initialConcurrency: schemas.anyNumber.optional(),
         maxRequestsPerMinute: schemas.anyNumber
             .refine((value) => Number.isInteger(value) || value === Infinity, 'Expected an integer or infinite number')
             .refine((value) => value >= 1, 'Expected a number greater than or equal to 1')
@@ -987,6 +995,7 @@ export class BasicCrawler<
             // AutoscaledPool shorthands
             minConcurrency,
             maxConcurrency,
+            initialConcurrency,
             maxRequestsPerMinute,
 
             blockedStatusCodes: blockedStatusCodesInput,
@@ -1011,12 +1020,15 @@ export class BasicCrawler<
         // hammering a site.
         if (
             concurrencySystem !== undefined &&
-            (minConcurrency !== undefined || maxConcurrency !== undefined || maxRequestsPerMinute !== undefined)
+            (minConcurrency !== undefined ||
+                maxConcurrency !== undefined ||
+                initialConcurrency !== undefined ||
+                maxRequestsPerMinute !== undefined)
         ) {
             throw new Error(
-                'The `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts cannot be combined with ' +
-                    '`concurrencySystem` - they configure the default `ConcurrencySystem` that a supplied one ' +
-                    'replaces. Pass them to the `ConcurrencySystem` constructor instead.',
+                'The `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts ' +
+                    'cannot be combined with `concurrencySystem` - they configure the default `ConcurrencySystem` ' +
+                    'that a supplied one replaces. Pass them to the `ConcurrencySystem` constructor instead.',
             );
         }
 
@@ -1348,6 +1360,9 @@ export class BasicCrawler<
                         minConcurrency,
                         maxConcurrency,
                         maxTasksPerMinute: maxRequestsPerMinute,
+                        // Spread conditionally - an explicit `undefined` would clobber a subclass default, see
+                        // `HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS`.
+                        ...(initialConcurrency !== undefined && { desiredConcurrency: initialConcurrency }),
                         log: this.log,
                     }),
                 );
@@ -1358,7 +1373,7 @@ export class BasicCrawler<
 
     /**
      * Builds the crawler-owned default {@apilink ConcurrencySystem} from the resolved
-     * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
+     * `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
      * {@apilink BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
      *
      * Subclasses may override this to tune the default system (e.g. {@apilink HttpCrawler} raises the starting

test/core/crawlers/basic_crawler.test.ts

@@ -276,7 +276,7 @@ describe('BasicCrawler', () => {
         expect(bookedFor).toEqual(new Set(['crawler-a', 'crawler-b']));
     });
 
-    test.each(['minConcurrency', 'maxConcurrency', 'maxRequestsPerMinute'] as const)(
+    test.each(['minConcurrency', 'maxConcurrency', 'initialConcurrency', 'maxRequestsPerMinute'] as const)(
         'throws when %s is combined with a supplied concurrencySystem',
         (shortcut) => {
             expect(
@@ -624,6 +624,7 @@ describe('BasicCrawler', () => {
         const collect = (crawler: BasicCrawler) => ({
             minConcurrency: (crawler.concurrencySystem! as ConcurrencySystem).minConcurrency,
             maxConcurrency: (crawler.concurrencySystem! as ConcurrencySystem).maxConcurrency,
+            desiredConcurrency: (crawler.concurrencySystem! as ConcurrencySystem).desiredConcurrency,
             // eslint-disable-next-line dot-notation -- private member on the governor
             maxTasksPerMinute: (crawler.concurrencySystem! as ConcurrencySystem)['maxTasksPerMinute'],
         });
@@ -634,11 +635,17 @@ describe('BasicCrawler', () => {
             requestHandler,
             minConcurrency: 123,
             maxConcurrency: 456,
+            initialConcurrency: 234,
             maxRequestsPerMinute: 789,
         });
 
         // An injected system carries its own config (the shortcuts are rejected alongside one, see above).
-        const injectedSystem = new ConcurrencySystem({ minConcurrency: 16, maxConcurrency: 32, maxTasksPerMinute: 64 });
+        const injectedSystem = new ConcurrencySystem({
+            minConcurrency: 16,
+            maxConcurrency: 32,
+            desiredConcurrency: 24,
+            maxTasksPerMinute: 64,
+        });
         const injected = new BasicCrawler({
             requestList,
             requestHandler,
@@ -650,8 +657,18 @@ describe('BasicCrawler', () => {
         await Promise.all([shortcuts.run(), injected.run()]);
         await injectedSystem.stop();
 
-        expect(collect(shortcuts)).toEqual({ minConcurrency: 123, maxConcurrency: 456, maxTasksPerMinute: 789 });
-        expect(collect(injected)).toEqual({ minConcurrency: 16, maxConcurrency: 32, maxTasksPerMinute: 64 });
+        expect(collect(shortcuts)).toEqual({
+            minConcurrency: 123,
+            maxConcurrency: 456,
+            desiredConcurrency: 234,
+            maxTasksPerMinute: 789,
+        });
+        expect(collect(injected)).toEqual({
+            minConcurrency: 16,
+            maxConcurrency: 32,
+            desiredConcurrency: 24,
+            maxTasksPerMinute: 64,
+        });
         // The injected system is the very instance the pool uses.
         expect(injected.concurrencySystem!).toBe(injectedSystem);
     });

test/core/crawlers/http_crawler.test.ts

@@ -193,6 +193,18 @@ test('concurrency shortcuts coexist with the HTTP-optimized defaults', async ()
     expect(crawler.asConfigured!.desiredConcurrency).toBe(5);
 });
 
+test('initialConcurrency overrides the HTTP-optimized starting concurrency', async () => {
+    const crawler = new ObservableHttpCrawler({
+        initialConcurrency: 3,
+        maxRequestRetries: 0,
+        requestHandler: () => {},
+    });
+
+    await crawler.run([url]);
+
+    expect(crawler.asConfigured!.desiredConcurrency).toBe(3);
+});
+
 test('parseWithCheerio works', async () => {
     const results: string[] = [];