[pull] master from apify:master

docs/public-api/crawlee-basic.api.md

@@ -41,7 +41,7 @@ import type { StatisticState } from '@crawlee/core';
 import type { StorageBackend } from '@crawlee/types';
 import type { StorageIdentifier } from '@crawlee/core';
 import { StorageWritePolicy } from '@crawlee/core';
-import type { TaskLoopPredicates } from '@crawlee/core';
+import type { TaskLoopOptions } from '@crawlee/core';
 import { TimeoutError } from '@apify/timeout';
 import type { TypedRequestsLike } from '@crawlee/core';
 
@@ -151,7 +151,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
     statusMessageCallback?: StatusMessageCallback;
     statusMessageLoggingInterval?: number;
     storageBackend?: StorageBackend;
-    taskLoopOptions?: TaskLoopPredicates;
+    taskLoopOptions?: TaskLoopOptions;
     transactionalStorage?: boolean | Partial<StorageWritePolicy>;
 }
 

docs/public-api/crawlee-browser-pool.api.md

@@ -88,6 +88,7 @@ export abstract class BrowserController<Library extends CommonLibrary = CommonLi
     protected abstract _setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
     // (undocumented)
     totalPages: number;
+    waitForActive(): Promise<void>;
 }
 
 // @public (undocumented)

docs/public-api/crawlee-core.api.md

@@ -812,6 +812,8 @@ export interface KeyConsumer {
 export class KeyValueStore {
     [Symbol.asyncIterator]<T = unknown>(): AsyncGenerator<[string, T], void, undefined>;
     // (undocumented)
+    readonly backend: KeyValueStoreBackend;
+    // (undocumented)
     readonly configuration: Configuration;
     drop(): Promise<void>;
     entries<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<[string, T]> & Promise<[string, T][]>;
@@ -2063,6 +2065,11 @@ export interface SystemInfo {
     storageBackendInfo: LoadSignalInfo;
 }
 
+// @public (undocumented)
+export interface TaskLoopOptions extends TaskLoopPredicates {
+    maybeRunIntervalSecs?: number;
+}
+
 // @public
 export interface TaskLoopPredicates {
     isFinishedFunction?: () => Promise<boolean>;

oxlint.config.ts

@@ -36,7 +36,7 @@ export default defineConfig({
         // Enforces the naming convention from #3108: no `_`-prefixed members; private
         // properties use native `#` fields instead. The allow list covers the template-method
         // hooks that collide with their public wrappers, platform contracts (`Readable._read`),
-        // and documented internals (`__crawlee`, `_currentConcurrency`).
+        // and documented internals (`__crawlee`, `__purged`).
         'no-underscore-dangle': [
             'error',
             {
@@ -50,7 +50,6 @@ export default defineConfig({
                     '_getCookies',
                     '_setCookies',
                     '_read',
-                    '_currentConcurrency',
                     '__crawlee',
                     '__purged',
                     '__originalHistory__',

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -27,7 +27,7 @@ import type {
     StatisticState,
     StorageIdentifier,
     StorageWritePolicy,
-    TaskLoopPredicates,
+    TaskLoopOptions,
     TypedRequestsLike,
     UrlPatternObject,
 } from '@crawlee/core';
@@ -101,7 +101,7 @@ import type { ReadonlyDeep } from 'type-fest';
 import { z } from 'zod';
 
 import { LruCache } from '@apify/datastructures';
-import { addTimeoutToPromise, extendTimeout, TimeoutError } from '@apify/timeout';
+import { addTimeoutToPromise, extendTimeout, TimeoutError, tryCancel } from '@apify/timeout';
 import { cryptoRandomObjectId } from '@apify/utilities';
 
 import {
@@ -380,7 +380,7 @@ export interface BasicCrawlerOptions<
      * Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`
      * shortcuts, or a {@apilink BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.
      */
-    taskLoopOptions?: TaskLoopPredicates;
+    taskLoopOptions?: TaskLoopOptions;
 
     /**
      * A pre-configured concurrency governor — the component that decides whether there is free compute for one more
@@ -847,8 +847,7 @@ export class BasicCrawler<
     protected readonly requestHandler!: RequestHandler<ExtendedContext>;
     readonly #errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
     readonly #failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
-    // kept as TS-private: tests read it at runtime
-    private requestHandlerTimeoutMillis!: number;
+    #requestHandlerTimeoutMillis!: number;
     protected readonly internalTimeoutMillis: number;
     readonly #maxRequestRetries: number;
     readonly #maxCrawlDepth?: number;
@@ -870,8 +869,7 @@ export class BasicCrawler<
      * {@apilink ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
      * settable here.
      */
-    // kept as TS-private: tests mutate it at runtime
-    private taskLoopOptions: Omit<AutoscaledPoolOptions, 'concurrencySystem' | 'consumer'>;
+    #taskLoopOptions: Omit<AutoscaledPoolOptions, 'concurrencySystem' | 'consumer'>;
     protected readonly httpClient: BaseHttpClient;
     protected readonly retryOnBlocked: boolean;
     #respectRobotsTxtFile: boolean | { userAgent?: string };
@@ -1106,9 +1104,9 @@ export class BasicCrawler<
             this.#errorHandler = errorHandler;
 
             if (requestHandlerTimeoutSecs) {
-                this.requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
+                this.#requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
             } else {
-                this.requestHandlerTimeoutMillis = 60_000;
+                this.#requestHandlerTimeoutMillis = 60_000;
             }
 
             this.retryOnBlocked = retryOnBlocked;
@@ -1125,7 +1123,7 @@ export class BasicCrawler<
             // allow at least 5min for internal timeouts
             this.internalTimeoutMillis =
                 serviceLocator.getConfiguration().internalTimeoutMillis ??
-                Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
+                Math.max(this.#requestHandlerTimeoutMillis * 2, 300e3);
 
             this.#maxRequestRetries = maxRequestRetries;
             this.#maxCrawlDepth = maxCrawlDepth;
@@ -1171,13 +1169,13 @@ export class BasicCrawler<
             this.blockedStatusCodes = new Set(blockedStatusCodesInput ?? BLOCKED_STATUS_CODES);
 
             const maxSignedInteger = 2 ** 31 - 1;
-            if (this.requestHandlerTimeoutMillis > maxSignedInteger) {
+            if (this.#requestHandlerTimeoutMillis > maxSignedInteger) {
                 this.log.warning(
-                    `requestHandlerTimeoutMillis ${this.requestHandlerTimeoutMillis}` +
+                    `requestHandlerTimeoutMillis ${this.#requestHandlerTimeoutMillis}` +
                         ` does not fit a signed 32-bit integer. Limiting the value to ${maxSignedInteger}`,
                 );
 
-                this.requestHandlerTimeoutMillis = maxSignedInteger;
+                this.#requestHandlerTimeoutMillis = maxSignedInteger;
             }
 
             this.internalTimeoutMillis = Math.min(this.internalTimeoutMillis, maxSignedInteger);
@@ -1195,7 +1193,9 @@ export class BasicCrawler<
                 isFinishedFunction = async () => false;
             }
 
-            const crawlerOwnedTaskLoopConfiguration: Partial<typeof this.taskLoopOptions> = {
+            const crawlerOwnedTaskLoopConfiguration: Partial<
+                Omit<AutoscaledPoolOptions, 'concurrencySystem' | 'consumer'>
+            > = {
                 runTaskFunction: async () => {
                     const source = this.requestManager;
                     if (!source) throw new Error('Request provider is not initialized!');
@@ -1340,7 +1340,7 @@ export class BasicCrawler<
                 log: this.log,
             };
 
-            this.taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
+            this.#taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
 
             this.#resolveConcurrencySystem = () =>
                 OwnedOrInjected.resolve<IConcurrencySystem, ConcurrencySystem>(concurrencySystem, () =>
@@ -1899,7 +1899,7 @@ export class BasicCrawler<
         // which routes a run will hit, so reserve for the longest one any route asked for. The hint is
         // raise-only, so erring high here is safe.
         const maxRouteTimeoutSecs = (this.requestHandler as Partial<RouterHandler>).getMaxTimeoutSecs?.() ?? 0;
-        const handlerTimeoutSecs = Math.max(this.requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
+        const handlerTimeoutSecs = Math.max(this.#requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
 
         await requestManager.setExpectedRequestProcessingTimeSecs?.(Math.max(handlerTimeoutSecs + 5, 60));
     }
@@ -2179,6 +2179,7 @@ export class BasicCrawler<
         data: Parameters<Dataset['pushData']>[0],
         datasetIdentifier?: string | StorageIdentifier,
     ): Promise<void> {
+        tryCancel();
         const dataset = await this.getDataset(datasetIdentifier);
         return dataset.pushData(data);
     }
@@ -2289,7 +2290,7 @@ export class BasicCrawler<
         await this.#concurrencySystemDep.ifOwned((system) => system.start());
 
         this.#autoscaledPool = new AutoscaledPool({
-            ...this.taskLoopOptions,
+            ...this.#taskLoopOptions,
             concurrencySystem: this.#concurrencySystemDep.value,
             consumer: this.#identity,
         });
@@ -2329,7 +2330,7 @@ export class BasicCrawler<
      */
     private resolveRequestHandlerTimeoutMillis(
         label: string | undefined,
-        fallbackMillis = this.requestHandlerTimeoutMillis,
+        fallbackMillis = this.#requestHandlerTimeoutMillis,
     ): number {
         return this.getRouteTimeoutMillis(label) ?? fallbackMillis;
     }

packages/browser-pool/src/abstract-classes/browser-controller.ts

@@ -146,11 +146,17 @@ export abstract class BrowserController<
 
     #activate!: () => void;
 
-    // kept as TS-private: `BrowserPool` awaits it through cross-object bracket access
-    private isActivePromise = new Promise<void>((resolve) => {
+    #isActivePromise = new Promise<void>((resolve) => {
         this.#activate = resolve;
     });
 
+    /**
+     * Returns a promise that resolves once the browser controller has been activated.
+     */
+    async waitForActive(): Promise<void> {
+        await this.#isActivePromise;
+    }
+
     #commitBrowser!: () => void;
 
     #hasBrowserPromise = new Promise<void>((resolve) => {
@@ -238,7 +244,7 @@ export abstract class BrowserController<
     async newPage(pageOptions?: NewPageOptions): Promise<NewPageResult> {
         this.activePages++;
         this.totalPages++;
-        await this.isActivePromise;
+        await this.#isActivePromise;
         const page = await this._newPage(pageOptions);
         tryCancel();
         this.lastPageOpenedAt = Date.now();

packages/browser-pool/src/browser-pool.ts

@@ -599,8 +599,7 @@ export class BrowserPool<
     ) {
         // This is needed for concurrent newPage calls to wait for the browser launch.
         // It's not ideal though, we need to come up with a better API.
-        // eslint-disable-next-line dot-notation -- accessing private property
-        await browserController['isActivePromise'];
+        await browserController.waitForActive();
         tryCancel();
 
         const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined;

packages/core/src/autoscaling/autoscaled_pool.ts

@@ -26,7 +26,7 @@ const autoscaledPoolOptionsSchema = z.strictObject({
     log: validators.logger.default(() => serviceLocator.getLogger()),
     concurrencySystem: schemas.anyObject,
     consumer: schemas.anyObject.refine(
-        (value) => typeof value.id === 'string' && value.id.length > 0,
+        (value) => typeof value?.id === 'string' && value.id.length > 0,
         "Expected an object with a non-empty string 'id'",
     ),
 });
@@ -53,8 +53,16 @@ export interface TaskLoopPredicates {
     isFinishedFunction?: () => Promise<boolean>;
 }
 
+export interface TaskLoopOptions extends TaskLoopPredicates {
+    /**
+     * How often the pool should check if a new task is ready, in seconds.
+     * @default 0.5
+     */
+    maybeRunIntervalSecs?: number;
+}
+
 /** @internal */
-export interface AutoscaledPoolOptions extends TaskLoopPredicates {
+export interface AutoscaledPoolOptions extends TaskLoopOptions {
     /**
      * The governor that decides whether there is free compute for one more task. Typically a
      * {@apilink ConcurrencySystem}, but any {@apilink IConcurrencySystem} works. Share a single instance across
@@ -77,13 +85,6 @@ export interface AutoscaledPoolOptions extends TaskLoopPredicates {
      */
     runTaskFunction?: () => Promise<unknown>;
 
-    /**
-     * Indicates how often the pool should call the `runTaskFunction()` to start a new task, in seconds.
-     * This has no effect on starting new tasks immediately after a task completes.
-     * @default 0.5
-     */
-    maybeRunIntervalSecs?: number;
-
     /**
      * Timeout in which the `runTaskFunction` needs to finish, given in seconds.
      * @default 0

packages/core/src/autoscaling/concurrency_system.ts

@@ -222,8 +222,7 @@ export class ConcurrencySystem implements IConcurrencySystem {
     #minConcurrency: number;
     #maxConcurrency: number;
     #desiredConcurrency: number;
-    // kept as TS-private _-prefixed: autoscaled_pool tests write this backing field directly
-    private _currentConcurrency = 0;
+    #currentConcurrency = 0;
     #lastLoggingTime?: number;
     #tasksPerMinute: number[] = Array.from({ length: 60 }, () => 0);
 
@@ -360,7 +359,7 @@ export class ConcurrencySystem implements IConcurrencySystem {
     }
 
     get currentConcurrency(): number {
-        return this._currentConcurrency;
+        return this.#currentConcurrency;
     }
 
     /** Whether the system is currently monitoring load and autoscaling the budget. */
@@ -466,14 +465,14 @@ export class ConcurrencySystem implements IConcurrencySystem {
     hasCapacityForTask(_consumer?: ConcurrencyConsumer): boolean {
         this.warnIfNotRunning();
 
-        if (this._currentConcurrency >= this.#desiredConcurrency) {
+        if (this.#currentConcurrency >= this.#desiredConcurrency) {
             this.log.perf('Task will not run. Desired concurrency achieved.');
             return false;
         }
 
         const currentStatus = this.systemStatus.getCurrentStatus();
         const { isSystemIdle } = currentStatus;
-        if (!isSystemIdle && this._currentConcurrency >= this.#minConcurrency) {
+        if (!isSystemIdle && this.#currentConcurrency >= this.#minConcurrency) {
             this.log.perf(
                 'Task will not be run. System is overloaded.',
                 currentStatus as unknown as Record<string, unknown>,
@@ -512,14 +511,14 @@ export class ConcurrencySystem implements IConcurrencySystem {
             return false;
         }
 
-        this._currentConcurrency++;
+        this.#currentConcurrency++;
         this.#tasksPerMinute[0]++;
         return true;
     }
 
     /** Returns a slot to the shared budget, whoever booked it. */
     registerTaskEnd(_consumer?: ConcurrencyConsumer): void {
-        this._currentConcurrency--;
+        this.#currentConcurrency--;
     }
 
     /**
@@ -542,7 +541,7 @@ export class ConcurrencySystem implements IConcurrencySystem {
         const { isSystemIdle } = systemStatus;
         const weAreNotAtMax = this.#desiredConcurrency < this.#maxConcurrency;
         const minCurrentConcurrency = Math.floor(this.#desiredConcurrency * this.desiredConcurrencyRatio);
-        const weAreReachingDesiredConcurrency = this._currentConcurrency >= minCurrentConcurrency;
+        const weAreReachingDesiredConcurrency = this.#currentConcurrency >= minCurrentConcurrency;
 
         if (isSystemIdle && weAreNotAtMax && weAreReachingDesiredConcurrency) this.scaleUp(systemStatus);
 
@@ -559,7 +558,7 @@ export class ConcurrencySystem implements IConcurrencySystem {
             } else if (now > this.#lastLoggingTime + this.#loggingIntervalMillis) {
                 this.#lastLoggingTime = now;
                 this.log.info('state', {
-                    currentConcurrency: this._currentConcurrency,
+                    currentConcurrency: this.#currentConcurrency,
                     desiredConcurrency: this.#desiredConcurrency,
                     systemStatus,
                 });

packages/core/src/autoscaling/load_signal.ts

@@ -71,17 +71,15 @@ export interface LoadSignal {
 export class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
     #snapshots: T[] = [];
 
-    /** Retention window in milliseconds. Unbounded until {@apilink SnapshotStore.useSampleWindow|`useSampleWindow()`}. */
-    // kept as TS-private: concurrency_system tests read this retention window directly
-    private historyMillis = Infinity;
+    #historyMillis = Infinity;
 
     /**
      * Sizes retention to the window the signal will be sampled over, as handed to it in
      * {@apilink LoadSignal.start|`start()`}. Until this is called nothing is pruned at all, so a signal that ignores
      * its start context grows unboundedly.
      */
     useSampleWindow(maxSampleWindowMillis: number): void {
-        this.historyMillis = maxSampleWindowMillis;
+        this.#historyMillis = maxSampleWindowMillis;
     }
 
     /**
@@ -92,7 +90,7 @@ export class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
         let oldCount = 0;
         for (let i = 0; i < this.#snapshots.length; i++) {
             const { createdAt } = this.#snapshots[i];
-            if (now.getTime() - new Date(createdAt).getTime() > this.historyMillis) oldCount++;
+            if (now.getTime() - new Date(createdAt).getTime() > this.#historyMillis) oldCount++;
             else break;
         }
         if (oldCount) this.#snapshots.splice(0, oldCount);