diff --git a/docs/guides/result_storage.mdx b/docs/guides/result_storage.mdx index af17763e0fbc..cb7103ce52c9 100644 --- a/docs/guides/result_storage.mdx +++ b/docs/guides/result_storage.mdx @@ -172,7 +172,7 @@ The feature is escapable at three granularities: A few operations cannot be buffered, and silently letting them through would produce storage states that no rollback can undo. They throw inside a transaction, with an error pointing at `withDirectStorageAccess()`: -- `Dataset.drop()`, `KeyValueStore.drop()`, `RequestQueue.drop()` and `RequestQueue.purge()`, +- `drop()` and `purge()` on `Dataset`, `KeyValueStore` and `RequestQueue`, - the request queue processing internals (`fetchNextRequest()`, `markRequestAsHandled()`, `reclaimRequest()`), - `KeyValueStore.setValue()` with a **stream** value — a stream can only be consumed once, so it cannot serve both a read within the handler and the commit replay. Write streams under `withDirectStorageAccess()`. diff --git a/docs/package.json b/docs/package.json index f028fef954e5..ea2ed45c9be1 100644 --- a/docs/package.json +++ b/docs/package.json @@ -19,7 +19,7 @@ "@opentelemetry/api": "^1.9.0", "@opentelemetry/exporter-trace-otlp-grpc": "^0.210.0", "@opentelemetry/resources": "^2.0.0", - "@opentelemetry/sdk-node": "^0.210.0", + "@opentelemetry/sdk-node": "^0.217.0", "@opentelemetry/sdk-trace-base": "^2.0.0", "@opentelemetry/semantic-conventions": "^1.39.0", "apify": "*", diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md index 9b543bd6752d..f5dd7d1d987a 100644 --- a/docs/public-api/crawlee-basic.api.md +++ b/docs/public-api/crawlee-basic.api.md @@ -73,8 +73,7 @@ export class BasicCrawler; // (undocumented) protected getRobotsTxtFileForUrl(url: string): Promise; - // (undocumented) - hasFinishedBefore: boolean; + get hasFinishedBefore(): boolean; // (undocumented) protected readonly httpClient: BaseHttpClient; protected init(): Promise; @@ -170,7 +169,6 @@ export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult { // @public (undocumented) export interface CrawlerRunOptions extends CrawlerAddRequestsOptions { - purgeRequestQueue?: boolean; } // @public diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.md index f324e51f80b7..aab338a54dee 100644 --- a/docs/public-api/crawlee-core.api.md +++ b/docs/public-api/crawlee-core.api.md @@ -347,6 +347,7 @@ export class Dataset { // (undocumented) name?: string; static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise>; + purge(): Promise; pushData(data: Data | Data[]): Promise; reduce(iteratee: DatasetReducer): Promise; reduce(iteratee: DatasetReducer, memo: undefined, options: DatasetIteratorOptions): Promise; @@ -833,6 +834,7 @@ export class KeyValueStore { // (undocumented) readonly name?: string; static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise; + purge(): Promise; recordExists(key: string): Promise; static recordExists(key: string): Promise; setValue(key: string, value: T | null, options?: RecordOptions): Promise; diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md index 1156d1d17b0f..e151e40c28ec 100644 --- a/docs/upgrading/upgrading_v4.md +++ b/docs/upgrading/upgrading_v4.md @@ -21,7 +21,7 @@ This page summarizes the breaking changes in Crawlee v4. There are many, so the - **One concurrency budget for several crawlers.** The new [`ConcurrencySystem`](#autoscaling-moved-to-concurrencysystem) can be shared between crawlers, capping their combined concurrency instead of letting each one oversubscribe the host. - **Native `fetch` types.** HTTP clients and `context.response` now use the [standard `Response`](#crawlingcontextresponse-is-now-of-type-response), and `got-scraping` is an [opt-in dependency](#http-client-packages-and-basehttpclient-reshaped) instead of a mandatory one. - **The session is the rotation unit.** A session carries its proxy, cookies and error score, and is rotated as a whole when blocked — replacing [proxy tiers](#tieredproxyurls-is-removed-from-proxyconfiguration) and [session rotation counters](#maxsessionrotations-and-requestsessionrotationcount-are-removed). -- **Crawlers stop stepping on each other.** Multiple crawlers in one process [no longer share the default request queue](#multiple-crawler-instances-use-separate-default-request-queues), and repeated `run()` calls purge the queue instead of dropping and recreating it. +- **Crawlers stop stepping on each other.** Multiple crawlers in one process [no longer share the default request queue](#multiple-crawler-instances-use-separate-default-request-queues), and repeated `run()` calls [no longer empty it](#repeated-run-calls-no-longer-empty-the-request-queue) behind your back. - **Cookies behave.** `sendRequest` finally [respects your `Cookie` header](#cookie-handling-in-httpcrawler-and-sendrequest), and browser cookies set inside the handler are [persisted to the session](#browser-cookies-are-also-persisted-after-requesthandler). - **No half-written results.** Storage writes in a request handler are [transactional](#storage-writes-in-request-handlers-are-transactional) — a handler that throws leaves nothing behind, and its retry does not duplicate data. - **Simpler storage backend contract.** A custom storage backend is now [4 classes instead of 7](#storagebackend-interface-simplified). @@ -603,45 +603,28 @@ In v4, only the **first** crawler instance uses the default request queue. Each If you explicitly pass a `requestQueue` (or `requestManager`) to the crawler, that queue is used as-is regardless of instance order. -### Repeated `run()` calls use `purge()` instead of `drop()` + recreate +### Repeated `run()` calls no longer empty the request queue -When calling `crawler.run()` multiple times on the same crawler instance, v3 would drop the default request queue and create a fresh one between runs. In v4, the crawler **purges** the queue instead — clearing all requests and resetting internal counters, but keeping the same queue object. This is more efficient and avoids edge cases around stale references. +In v3, calling `crawler.run()` again on the same instance dropped the default request queue and created a fresh one, so the same URLs were crawled again — but only for a queue actually named `default`, which the Apify platform's default queue is not, so on the platform the second run silently crawled nothing. -The new `purge()` method is available on `RequestQueue` and is also defined as an optional method on the `IRequestManager` interface. +v4 does the same thing everywhere: nothing is emptied between runs. A repeated `run()` continues with the same request manager, and requests the previous run handled — a failed request counts as handled — are not processed again. Any crawl that ends up processing nothing while its request manager holds only handled requests warns and says why, instead of finishing silently; that also covers a second crawler sharing the queue, or a queue a previous process already worked through. -By default, only queues that the crawler created itself (the "owned" queue) are purged between runs — a user-supplied queue is never touched unless you explicitly opt in. The `purgeRequestQueue` option in `CrawlerRunOptions` controls this behavior: - -| `purgeRequestQueue` value | Owned queue (auto-created) | User-supplied queue | -|---|---|---| -| omitted (default) | Purged | Not purged | -| `true` | Purged | Purged | -| `false` | Not purged | Not purged | - -One combination has no sensible default: `sameDomainDelaySecs` over a request manager you supplied that does not pace on its own. The per-domain queues that have to be emptied are the crawler's, the manager underneath them is yours, and a purge cannot respect both — so a repeated `run()` throws and asks you to pass `purgeRequestQueue` explicitly rather than guessing. A manager that takes the delay as a floor has nothing of ours underneath it, and is left alone like any other supplied manager. +The `purgeRequestQueue` option of `crawler.run()` went away with the automatic purge. To crawl the same requests again, empty the queue yourself: ```typescript -// The purge happens automatically between run() calls: const crawler = new BasicCrawler({ requestHandler: async ({ request }) => { /* ... */ } }); await crawler.run(['https://example.com/a', 'https://example.com/b']); -// Queue is purged here, so the same URLs can be processed again: -await crawler.run(['https://example.com/a', 'https://example.com/c']); -``` -You can opt out of the automatic purge by passing `purgeRequestQueue: false`: +const queue = await crawler.getRequestQueue(); +await queue.purge?.(); -```typescript -await crawler.run(urls, { purgeRequestQueue: false }); +// The same URLs are crawled again: +await crawler.run(['https://example.com/a', 'https://example.com/c']); ``` -If you supplied your own `requestQueue` and want it purged between runs, pass `purgeRequestQueue: true` explicitly: +`purge()` — empty the storage, keep its id and name — is new in v4 and available on `Dataset`, `KeyValueStore` and `RequestQueue`, as well as being an optional method on the `IRequestManager` interface. -```typescript -const queue = await RequestQueue.open('my-queue'); -const crawler = new BasicCrawler({ requestQueue: queue, requestHandler: async () => { /* ... */ } }); -await crawler.run(['https://example.com/first']); -// Explicitly purge the user-supplied queue before the second run: -await crawler.run(['https://example.com/second'], { purgeRequestQueue: true }); -``` +This has nothing to do with `purgeOnStart` / `CRAWLEE_PURGE_ON_START`, which still wipes the default storages once per process before the first run. ### Storage `.open()` now also accepts `{ id?, name? }` diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index d6774f247b71..8c5bdfd3ae2c 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -102,7 +102,7 @@ import type { ReadonlyDeep } from 'type-fest'; import { z } from 'zod'; import { LruCache } from '@apify/datastructures'; -import { addTimeoutToPromise, extendTimeout, TimeoutError, tryCancel } from '@apify/timeout'; +import { addTimeoutToPromise, extendTimeout, storage as timeoutStorage, TimeoutError, tryCancel } from '@apify/timeout'; import { cryptoRandomObjectId } from '@apify/utilities'; import { @@ -843,9 +843,14 @@ export class BasicCrawler< } running = false; - hasFinishedBefore = false; + #hasFinishedBefore = false; #unexpectedStop = false; + /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */ + get hasFinishedBefore(): boolean { + return this.#hasFinishedBefore; + } + #log!: CrawleeLogger; get log(): CrawleeLogger { @@ -859,16 +864,6 @@ export class BasicCrawler< protected readonly internalTimeoutMillis: number; readonly #maxRequestRetries: number; readonly #maxCrawlDepth?: number; - /** - * How much of {@apilink BasicCrawler.requestManager} the crawler may empty between repeated `run()` calls. - * - * - `all` — nothing under it came from the caller, so one `purge()` on the outside covers everything. - * - `none` — the caller supplied it and the crawler put nothing of its own inside. - * - `ambiguous` — the caller supplied it, but `sameDomainDelaySecs` put the crawler's own per-domain queues - * underneath: purging empties the caller's storage too, skipping leaves ours stale. A repeated `run()` asks - * rather than guessing. - */ - readonly #purgeableExtent: 'all' | 'none' | 'ambiguous'; readonly #maxRequestsPerCrawl?: number; private get handledRequestsCount(): number { @@ -1105,14 +1100,6 @@ export class BasicCrawler< const pacerNeeded = sameDomainDelaySecs > 0 && !floorTaken; - // Our per-domain queues under a manager the caller owns is the case with no right answer; a floor - // it took leaves nothing of ours behind. See `#purgeableExtent`. - if (suppliedManager === undefined) { - this.#purgeableExtent = 'all'; - } else { - this.#purgeableExtent = pacerNeeded ? 'ambiguous' : 'none'; - } - // Built here rather than at first use so it can sit *inside* the tandem below, which is where a // loader's transferred requests pass through it. const writableManager = pacerNeeded @@ -1695,152 +1682,161 @@ export class BasicCrawler< * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for * running {@apilink BasicCrawler.addRequests|`crawler.addRequests()`} before {@apilink BasicCrawler.run|`crawler.run()`}. * + * Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous + * run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one + * if that is what you want. + * * @param [requests] The requests to add. - * @param [options] Options for the request queue. + * @param [options] Options for adding the initial requests. */ async run(requests?: TypedRequestsLike, options?: CrawlerRunOptions): Promise { - if (this.running) { - throw new Error( - 'This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.', - ); - } - - const { purgeRequestQueue, ...addRequestsOptions } = options ?? {}; - - if (this.hasFinishedBefore) { - // When executing the run method for the second time explicitly, - // we need to purge the RQ to allow processing the same requests again — this is important so users can - // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and - // ignored — as a failed request is still handled. - // `purgeRequestQueue` unset purges only storage the crawler opened itself (see `#purgeableExtent`); - // `true` also purges a caller-supplied manager, `false` purges nothing. - if (purgeRequestQueue === undefined && this.#purgeableExtent === 'ambiguous') { - throw new Error( - 'Cannot decide what to purge before running again: `sameDomainDelaySecs` paces the request ' + - 'manager you supplied, so the per-domain queues that have to be emptied are the ' + - "crawler's while the manager underneath them is yours. Say which you want: " + - '`run(requests, { purgeRequestQueue: true })` empties both, `false` empties neither.', - ); - } - - if (purgeRequestQueue !== false && (this.#purgeableExtent === 'all' || purgeRequestQueue === true)) { - // One call from the outside in reaches everything the manager wraps, a pacer's per-domain queues - // included. - await this.requestManager?.purge?.(); - } + // A crawl is the top level of its own transaction and timeout scope, not a participant in the caller's. + return withDirectStorageAccess(async () => + timeoutStorage.exit(async () => { + if (this.running) { + throw new Error( + 'This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.', + ); + } - // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built. - await this.#statisticsDep.ifOwned(async (stats) => { - stats.reset(); - await stats.resetStore(); - }); - await this.#sessionPoolDep.ifOwned((pool) => pool.resetStore()); - } + if (this.#hasFinishedBefore) { + // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built. + await this.#statisticsDep.ifOwned(async (stats) => { + stats.reset(); + await stats.resetStore(); + }); + await this.#sessionPoolDep.ifOwned((pool) => pool.resetStore()); + } - this.#unexpectedStop = false; - this.running = true; - this.#loggedPerRun.clear(); + this.#unexpectedStop = false; + this.running = true; + this.#loggedPerRun.clear(); - await purgeDefaultStorages({ - onlyPurgeOnce: true, - storageBackend: serviceLocator.getStorageBackend(), - configuration: serviceLocator.getConfiguration(), - }); + await purgeDefaultStorages({ + onlyPurgeOnce: true, + storageBackend: serviceLocator.getStorageBackend(), + configuration: serviceLocator.getConfiguration(), + }); - if (requests) { - await this.addRequests(requests, addRequestsOptions); - } + if (requests) { + await this.addRequests(requests, options); + } - try { - await this.init(); - await this.statistics.startCapturing(); - } catch (error) { - // Clean up here before propagating, otherwise a failed startup would leave the process hanging. - await this.teardown().catch((teardownError) => { - this.log.exception(teardownError as Error, 'Cleaning up after a failed crawler startup failed.'); - }); + try { + await this.init(); + await this.statistics.startCapturing(); + } catch (error) { + // Clean up here before propagating, otherwise a failed startup would leave the process hanging. + await this.teardown().catch((teardownError) => { + this.log.exception( + teardownError as Error, + 'Cleaning up after a failed crawler startup failed.', + ); + }); - // The run never began, so let the instance be run again instead of leaving it wedged as `running`. - this.running = false; - throw error; - } + // The run never began, so let the instance be run again instead of leaving it wedged as `running`. + this.running = false; + throw error; + } - const periodicLogger = this.getPeriodicLogger(); - this.setStatusMessage('Starting the crawler.', { level: 'INFO' }); + const periodicLogger = this.getPeriodicLogger(); + this.setStatusMessage('Starting the crawler.', { level: 'INFO' }); - const sigintHandler = async () => { - this.log.warning( - 'Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start', - ); - await this.pauseOnMigration(); - await this.#autoscaledPool!.abort(); - }; + const sigintHandler = async () => { + this.log.warning( + 'Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start', + ); + await this.pauseOnMigration(); + await this.#autoscaledPool!.abort(); + }; - // Attach a listener to handle migration and aborting events gracefully. - const boundPauseOnMigration = this.pauseOnMigration.bind(this); - process.once('SIGINT', sigintHandler); - const eventManager = serviceLocator.getEventManager(); - eventManager.on(EventType.MIGRATING, boundPauseOnMigration); - eventManager.on(EventType.ABORTING, boundPauseOnMigration); + // Attach a listener to handle migration and aborting events gracefully. + const boundPauseOnMigration = this.pauseOnMigration.bind(this); + process.once('SIGINT', sigintHandler); + const eventManager = serviceLocator.getEventManager(); + eventManager.on(EventType.MIGRATING, boundPauseOnMigration); + eventManager.on(EventType.ABORTING, boundPauseOnMigration); - let stats = {} as FinalStatistics; + let stats = {} as FinalStatistics; - try { - await this.#autoscaledPool!.run(); - } finally { - await this.statistics.stopCapturing(); - await this.teardown(); - - process.off('SIGINT', sigintHandler); - eventManager.off(EventType.MIGRATING, boundPauseOnMigration); - eventManager.off(EventType.ABORTING, boundPauseOnMigration); - - const finalStats = this.statistics.calculate(); - stats = { - requestsFinished: this.statistics.state.requestsFinished, - requestsFailed: this.statistics.state.requestsFailed, - retryHistogram: this.statistics.requestRetryHistogram, - ...finalStats, - }; - this.log.info('Final request statistics:', stats as unknown as Record); + try { + await this.#autoscaledPool!.run(); + } finally { + await this.statistics.stopCapturing(); + await this.teardown(); + + process.off('SIGINT', sigintHandler); + eventManager.off(EventType.MIGRATING, boundPauseOnMigration); + eventManager.off(EventType.ABORTING, boundPauseOnMigration); + + const finalStats = this.statistics.calculate(); + stats = { + requestsFinished: this.statistics.state.requestsFinished, + requestsFailed: this.statistics.state.requestsFailed, + retryHistogram: this.statistics.requestRetryHistogram, + ...finalStats, + }; + this.log.info('Final request statistics:', stats as unknown as Record); + + // A crawl that did nothing while the manager holds only handled requests is a mistake whoever + // handled them - this run, another crawler on the same queue, or a previous process. Starting + // against handled requests is not: that is what resuming a crawl looks like. + if (stats.requestsFinished + stats.requestsFailed === 0) { + // Never let the diagnostic itself break the run. + const alreadyHandled = (await this.requestManager?.getHandledCount().catch(() => 0)) ?? 0; + + if (alreadyHandled > 0) { + this.log.warningOnce( + 'This crawl processed no requests - the request manager holds ' + + `${alreadyHandled} request${alreadyHandled === 1 ? '' : 's'}, all of them ` + + 'already handled, and a failed request counts as handled too. Nothing ' + + 'empties a queue between runs, so to crawl them again, purge it ' + + '(`await queue.purge()`) or use a fresh one (e.g. ' + + '`RequestQueue.open({ alias: "second-run" })`) with a freshly created ' + + 'crawler instance.', + ); + } + } - if (this.statistics.errorTracker.total !== 0) { - const prettify = ([count, info]: [number, string[]]) => - `${count}x: ${info.at(-1)!.trim()} (${info[0]})`; + if (this.statistics.errorTracker.total !== 0) { + const prettify = ([count, info]: [number, string[]]) => + `${count}x: ${info.at(-1)!.trim()} (${info[0]})`; - this.log.info(`Error analysis:`, { - totalErrors: this.statistics.errorTracker.total, - uniqueErrors: this.statistics.errorTracker.getUniqueErrorCount(), - mostCommonErrors: this.statistics.errorTracker.getMostPopularErrors(3).map(prettify), - }); - } + this.log.info(`Error analysis:`, { + totalErrors: this.statistics.errorTracker.total, + uniqueErrors: this.statistics.errorTracker.getUniqueErrorCount(), + mostCommonErrors: this.statistics.errorTracker.getMostPopularErrors(3).map(prettify), + }); + } - const client = serviceLocator.getStorageBackend(); + const client = serviceLocator.getStorageBackend(); - if (client.teardown) { - let finished = false; - setTimeout(() => { - if (!finished) { - this.log.info('Waiting for the storage to write its state to file system.'); + if (client.teardown) { + let finished = false; + setTimeout(() => { + if (!finished) { + this.log.info('Waiting for the storage to write its state to file system.'); + } + }, 1000); + await client.teardown(); + finished = true; } - }, 1000); - await client.teardown(); - finished = true; - } - periodicLogger.stop(); - this.setStatusMessage( - `Finished! Total ${this.statistics.state.requestsFinished + this.statistics.state.requestsFailed} requests: ${ - this.statistics.state.requestsFinished - } succeeded, ${this.statistics.state.requestsFailed} failed.`, - { isStatusMessageTerminal: true, level: 'INFO' }, - ); + periodicLogger.stop(); + this.setStatusMessage( + `Finished! Total ${this.statistics.state.requestsFinished + this.statistics.state.requestsFailed} requests: ${ + this.statistics.state.requestsFinished + } succeeded, ${this.statistics.state.requestsFailed} failed.`, + { isStatusMessageTerminal: true, level: 'INFO' }, + ); - this.running = false; - this.hasFinishedBefore = true; - } + this.running = false; + this.#hasFinishedBefore = true; + } - return stats; + return stats; + }), + ); } /** @@ -3083,19 +3079,7 @@ export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions, En export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {} -export interface CrawlerRunOptions extends CrawlerAddRequestsOptions { - /** - * Controls whether the request queue is purged between repeated `run()` calls on the same crawler instance. - * Purging clears all requests and resets internal counters, allowing the same URLs to be processed again. - * - * - **`undefined`** (default) — only the crawler's own (auto-created) queue is purged. - * A user-supplied `requestQueue` is left untouched. - * - **`true`** — the queue is always purged, even if it was supplied by the user. - * - **`false`** — nothing is purged. Only genuinely new requests will be processed; - * note that even a failed request is considered handled. - */ - purgeRequestQueue?: boolean; -} +export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {} /** The hostname of `url`, falling back to the whole string when it is not parseable - for log messages only. */ function hostnameOrUrl(url: string): string { diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.ts index b2da8b4eb825..c8b35973b65b 100644 --- a/packages/core/src/storages/dataset.ts +++ b/packages/core/src/storages/dataset.ts @@ -783,6 +783,16 @@ export class Dataset { serviceLocator.getStorageInstanceManager().removeFromCache(this); } + /** + * Removes all items from the dataset but keeps the dataset itself, along with its + * {@apilink Dataset.id|`id`} and {@apilink Dataset.name|`name`}. + */ + async purge(): Promise { + rejectOperationInTransaction('Dataset.purge()'); + + await this.backend.purge(); + } + /** * Opens a dataset and returns a promise resolving to an instance of the {@apilink Dataset} class. * diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.ts index 9b075fa54a99..8e3520172c86 100644 --- a/packages/core/src/storages/key_value_store.ts +++ b/packages/core/src/storages/key_value_store.ts @@ -619,6 +619,18 @@ export class KeyValueStore { serviceLocator.getStorageInstanceManager().removeFromCache(this); } + /** + * Removes all records from the store but keeps the store itself, along with its + * {@apilink KeyValueStore.id|`id`} and {@apilink KeyValueStore.name|`name`}. + */ + async purge(): Promise { + rejectOperationInTransaction('KeyValueStore.purge()'); + + await this.backend.purge(); + // The auto-saved values this cache holds are no longer in the store. + this.#cache.clear(); + } + /** @internal */ clearCache(): void { rejectOperationInTransaction('KeyValueStore.clearCache()'); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 51a3987ee392..db85dff5695a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -210,12 +210,8 @@ catalogs: version: 4.4.3 overrides: - playwright-core: 1.61.1 - '@browserbasehq/stagehand': 3.0.7 minimatch: ^9.0.0 - tmp: ^0.2.7 '@puppeteer/browsers': ^3.0.4 - form-data: ^4.0.6 tar: ^7.5.16 lerna>js-yaml: ^4.2.0 lerna>minimatch: ^3.1.4 @@ -519,8 +515,8 @@ importers: specifier: ^2.0.0 version: 2.10.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-node': - specifier: ^0.210.0 - version: 0.210.0(@opentelemetry/api@1.9.0)(supports-color@8.1.1) + specifier: ^0.217.0 + version: 0.217.0(@opentelemetry/api@1.9.0)(supports-color@8.1.1) '@opentelemetry/sdk-trace-base': specifier: ^2.0.0 version: 2.10.0(@opentelemetry/api@1.9.0) @@ -1154,7 +1150,7 @@ importers: devDependencies: '@browserbasehq/stagehand': specifier: 3.0.7 - version: 3.0.7(@cfworker/json-schema@4.1.1)(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(supports-color@8.1.1)(zod@4.4.3) + version: 3.0.7(@cfworker/json-schema@4.1.1)(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(supports-color@8.1.1)(zod@4.4.3) playwright: specifier: ^1.60.0 version: 1.61.1 @@ -3844,12 +3840,16 @@ packages: resolution: {integrity: sha512-CMtLxp+lYDriveZejpBND/2TmadrrhUfChyxzmkFtHaMDdSKfP59MAYyA0ICBvEBdm3iXwLcaj/8Ic/pnGw9Yg==} engines: {node: '>=8.0.0'} + '@opentelemetry/api-logs@0.217.0': + resolution: {integrity: sha512-Cdq0jW2lknrNfrAm92MyEAvpe2cRsKjdnQLHUL6xRA4IVUnsWx6P65E7NcUO0Y+L4w1Aee5iV8FvjSwd+lrs9A==} + engines: {node: '>=8.0.0'} + '@opentelemetry/api@1.9.0': resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} engines: {node: '>=8.0.0'} - '@opentelemetry/configuration@0.210.0': - resolution: {integrity: sha512-tM0ROS/hZM72kB55cSjDcghVcUXBJdGkGzpkhD7M1B/gpcvZPSGfjFgKN3dgmxNgF76NxtbUwv3ik0wS+Kz52g==} + '@opentelemetry/configuration@0.217.0': + resolution: {integrity: sha512-xCtrYOhBqdy6ZOMfe0Oa73ZKF+2LMhoOv4L5vmwAHVvOXUg+V3fvKuEIr9ZyD0Ow+vxllEjWO6PV1wd0DOtyvw==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': ^1.9.0 @@ -3860,8 +3860,8 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/context-async-hooks@2.4.0': - resolution: {integrity: sha512-jn0phJ+hU7ZuvaoZE/8/Euw3gvHJrn2yi+kXrymwObEPVPjtwCmkvXDRQCWli+fCTTF/aSOtXaLr7CLIvv3LQg==} + '@opentelemetry/context-async-hooks@2.7.1': + resolution: {integrity: sha512-OPFBYuXEn1E4ja3Y6eeA7O+ZnLBNcXTV5Cgsn1VaqBZ6hC5FnpZPLBNme1LJY8ZtF4aOujPKFoeWN4ik487KuQ==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' @@ -3878,44 +3878,50 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/exporter-logs-otlp-grpc@0.210.0': - resolution: {integrity: sha512-+BolenqOO6ow65go7uWRYPvvs/BBIWp1mtRn93VvGduqvMVH/IY8nXrt80a4L9hZ7lHi2Tq2/NcC3H2QzcWKag==} + '@opentelemetry/core@2.7.1': + resolution: {integrity: sha512-QAqIj32AtK6+pEVNG7EOVxHdE06RP+FM5qpiEJ4RtDcFIqKUZHYhl7/7UY5efhwmwNAg7j8QbJVBLxMerc0+gw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/exporter-logs-otlp-grpc@0.217.0': + resolution: {integrity: sha512-vC5S0Dc+noxD86CVtNu1+awCHPA5Kewi1Sg23ps+9lh4YifwsKXh3pe4XTNEKtUJiAcjpJ5dqStGakLbrSE+YQ==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/exporter-logs-otlp-http@0.210.0': - resolution: {integrity: sha512-Q8/SEQtgrErbVVRg9M9iaG8m5wdPNdU0UOF7U43sAhwfmPG92ZOk/aenKhg0DXSNJHhkCDNCgS1kSoErAB3z0A==} + '@opentelemetry/exporter-logs-otlp-http@0.217.0': + resolution: {integrity: sha512-KfLAdt1uilVE+3FxbgVnp2ZrzqbIawzcesnRoi+Kh9ckB5Ld5D8btUgoBvwTbdmuNx1j6b132Wsh72azq+pPNQ==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/exporter-logs-otlp-proto@0.210.0': - resolution: {integrity: sha512-Y/yPc+gDhsWB7AsNzQWxblw4ULbvhCycMaQ2aAn+HSAVbgbMiZa0SbclPVHSnpnNzKSLVavFjweAr0pQA1KKLg==} + '@opentelemetry/exporter-logs-otlp-proto@0.217.0': + resolution: {integrity: sha512-Se0GG/ZO24mQTlQj7zprR4pNI0nKe4lPDPBsuJmi6508b9TlZEuUd3EfyuHk6oJxzL7fGyDFYAbxNigQvRP2ZQ==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/exporter-metrics-otlp-grpc@0.210.0': - resolution: {integrity: sha512-pWZ/Tjrqev9rdkqe8F6A9FGddLZrjl6iRAU5LBvvRL6I3PSgG8z1xM0cESAy1jzAF4wGohnAh8rB7hHzpUOYEA==} + '@opentelemetry/exporter-metrics-otlp-grpc@0.217.0': + resolution: {integrity: sha512-0GpJKnCoVaVA1rKBMVPHziznfOQlXgH72S9ktjBAF1AnAVPzX7vVEBGrhwiSxxHDAiefXk+J8znApsMb/K6Z3w==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/exporter-metrics-otlp-http@0.210.0': - resolution: {integrity: sha512-JpLThG8Hh8A/Jzdzw9i4Ftu+EzvLaX/LouN+mOOHmadL0iror0Qsi3QWzucXeiUsDDsiYgjfKyi09e6sltytgA==} + '@opentelemetry/exporter-metrics-otlp-http@0.217.0': + resolution: {integrity: sha512-1zkMzzhiNJdVmLxuwkltqWGw4fOOam47bqRxmuQNjyKJe/9NmY5cIrZ4kiQV7sVGxoOgT0ZvGUfLcjvtpC/b9Q==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/exporter-metrics-otlp-proto@0.210.0': - resolution: {integrity: sha512-CFa7SOinYOVWIWJuQL7XFeyedzmFGIpHpSMNFE8Xefb6iGB4m+MukQecdssvPcJKYlfF5FpovEOLXwafAzsXWQ==} + '@opentelemetry/exporter-metrics-otlp-proto@0.217.0': + resolution: {integrity: sha512-nfxt/KxVGFkjkO/M+58y1ugHu/dwPtxG4eYq0KApcQ7xk5CHzhdn+IuLZfDSvNDrJ3Uy5q++Fj/wbK7i8yryfQ==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/exporter-prometheus@0.210.0': - resolution: {integrity: sha512-8i+7d70Hho6pcheTtbqIuS+bo+AIX/oNUTMwIEZoehUE4ZdbGmeVaE+hJS2LAErFeFaU71w164lAgYyMUEQ8zw==} + '@opentelemetry/exporter-prometheus@0.217.0': + resolution: {integrity: sha512-U9MCXxJu0sBCh5aEkylYRR4xVIL8D1CW6dGwvYXbfFr0qveSorfD0XJchCAWoW6QfAAIcY/yxjf4Dj8OgkHBPw==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': ^1.3.0 @@ -3926,20 +3932,26 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/exporter-trace-otlp-http@0.210.0': - resolution: {integrity: sha512-9JkyaCl70anEtuKZdoCQmjDuz1/paEixY/DWfsvHt7PGKq3t8/nQ/6/xwxHjG+SkPAUbo1Iq4h7STe7Pk2bc5A==} + '@opentelemetry/exporter-trace-otlp-grpc@0.217.0': + resolution: {integrity: sha512-fPZs2fw7veLH3pEKu8vSepUa2fQpAE2P7al6qU10aH9GrEJJ8YaPgsd5xON7by5rbcEVS71FOU2aWyK6nzB7VQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-trace-otlp-http@0.217.0': + resolution: {integrity: sha512-38YQoqtYjglz2GV94LGUN/djLvxtvGIQO68o6qAFPVshjmwSdX1F2i0c7vn3lEl1L5B/YqjB/bgKXaVx7KO+RQ==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/exporter-trace-otlp-proto@0.210.0': - resolution: {integrity: sha512-qVUY7Hsm/t5buGOtPcTV1Ch4W9kj2wGaQaAF5FO4XR8TMKl2GM45tUCnr0/1dF3wo4RG9khMxrddeQWdRL4fIg==} + '@opentelemetry/exporter-trace-otlp-proto@0.217.0': + resolution: {integrity: sha512-nPV8gKHUiSuTZpQcnZU3/pBlK7crSyEGpZuh5MtWySB0vv6NNG0QvvfKitQt+Fc2Mc6qfyU54KlZcurwoTbrVg==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/exporter-zipkin@2.4.0': - resolution: {integrity: sha512-qpiXY0TUEFjBBp9b1na9LfuVQw6W8LH+te7uv+CC+0Up78ZDtZZwOjK2M7CL7Nspnw+yS4JdgEA7oxsBu0Ctsg==} + '@opentelemetry/exporter-zipkin@2.7.1': + resolution: {integrity: sha512-mfsD9bKAxcKrh5+y08TPodvClBO0CznBE3p79YAGnO81WI4LrdsGA65T53e4iTSbCalW4WaUpkbeJcbpyIUHfg==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': ^1.0.0 @@ -3950,32 +3962,56 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 + '@opentelemetry/instrumentation@0.217.0': + resolution: {integrity: sha512-24ucQMjz7Y34Kw3trbxL2ZrssbtgWnR+Clpaa+YdeWuuyH3Cvk23Q03PcQvqiZrDvt8AmQmjgg9v6Y9PHoxG7w==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + '@opentelemetry/otlp-exporter-base@0.210.0': resolution: {integrity: sha512-uk78DcZoBNHIm26h0oXc8Pizh4KDJ/y04N5k/UaI9J7xR7mL8QcMcYPQG9xxN7m8qotXOMDRW6qTAyptav4+3w==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': ^1.3.0 + '@opentelemetry/otlp-exporter-base@0.217.0': + resolution: {integrity: sha512-eYfqnB3UhKu/5frhd1R6+FprKygbhkomuaceMXDyzxbfXB9tKgZOVmjaJ02CkLA6Tdzumxl+e2H+vo2a8jiMPQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + '@opentelemetry/otlp-grpc-exporter-base@0.210.0': resolution: {integrity: sha512-fEJs8UhkFMrdXMOCLXyKd2uc6N209tIi8IBNqSTi83ri+MlMFrBKnOtklmv9/zzxovoN5zD1waRt6XBFGPfmIw==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': ^1.3.0 + '@opentelemetry/otlp-grpc-exporter-base@0.217.0': + resolution: {integrity: sha512-7RTAdZuOsCDnsyqTCG4+bDzrfnsWdzkRs7z0AVi/V3tEQx0oKeyc+OuRWYxnRsmaJXgxcmB8vb/lfxn58Dj6Ag==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + '@opentelemetry/otlp-transformer@0.210.0': resolution: {integrity: sha512-nkHBJVSJGOwkRZl+BFIr7gikA93/U8XkL2EWaiDbj3DVjmTEZQpegIKk0lT8oqQYfP8FC6zWNjuTfkaBVqa0ZQ==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/propagator-b3@2.4.0': - resolution: {integrity: sha512-6VPsFiMUkJBre/86F0d+PZMaUCcuLA9DtZuC46KH8EeVEKZPEM2WlX35M/qmde8UpzoQL9qzdz54YjUYABt8Uw==} + '@opentelemetry/otlp-transformer@0.217.0': + resolution: {integrity: sha512-MKK8UHKFUOGAvbZRWh90MhwHG+Fxm6OROBdjKPCF+HQobjuJ/Kuf8Chs8CR45X1aqotxrMj7OxTdsXe8sXuGVA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/propagator-b3@2.7.1': + resolution: {integrity: sha512-RJid6E2CKyeGfKBzXKF21ejabGMHypFkPAh3qZ+NvI+SGjuIye79t3PmiqcDgtRzdKH6ynXzbfslQ8DfpRUg2A==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/propagator-jaeger@2.4.0': - resolution: {integrity: sha512-t6muBL/3AMD++1EMF658C/KIpj3gfmTmftX3mEQql4KIxNGFvacCmmTtrQt9IZAJmQRfjQRCkv+vsGbQugeJIw==} + '@opentelemetry/propagator-jaeger@2.7.1': + resolution: {integrity: sha512-KMjVBHzP4N60bOzxja76M1F1hZZ43lGPga5ix+mkv9+kk1nx9SbkxSvJsMbuVUxdPQmsPTqGShmhN8ulrMOg6Q==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' @@ -3992,20 +4028,38 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/resources@2.7.1': + resolution: {integrity: sha512-DeT6KKolmC4e/dRQvMQ/RwlnzhaqeiFOXY5ngoOPJ07GgVVKxZOg9EcrNZb5aTzUn+iCrJldAgOfQm1O/QfPAQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/sdk-logs@0.210.0': resolution: {integrity: sha512-YuaL92Dpyk/Kc1o4e9XiaWWwiC0aBFN+4oy+6A9TP4UNJmRymPMEX10r6EMMFMD7V0hktiSig9cwWo59peeLCQ==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': '>=1.4.0 <1.10.0' + '@opentelemetry/sdk-logs@0.217.0': + resolution: {integrity: sha512-BB+PcHItcZDL63dPMW+mJvwN9rk37wuIDjRxbVlg6pPDvDR/7GL7UJHbGsllgoggOoTimsKgENaWPoGch/oE1A==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.4.0 <1.10.0' + '@opentelemetry/sdk-metrics@2.4.0': resolution: {integrity: sha512-qSbfq9mXbLMqmPEjijl32f3ZEmiHekebRggPdPjhHI6t1CsAQOR2Aw/SuTDftk3/l2aaPHpwP3xM2DkgBA1ANw==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': '>=1.9.0 <1.10.0' - '@opentelemetry/sdk-node@0.210.0': - resolution: {integrity: sha512-KymqUtYvfpblDNgGxBXYqCcDjYXwjOF7Muc6ocs0rMlG/66Hcs9KiJ7hg4zLOv63JubF/vxi5WXaLrQrPKyaZQ==} + '@opentelemetry/sdk-metrics@2.7.1': + resolution: {integrity: sha512-MpDJdkiFDs3Pm1RHO3KByuZbuBdJEXEAkiC0+yJdsZGVCdf1RpHR6n+LHDcS7ffmfrt5kVCzJSCfm4z2C7v0uQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.9.0 <1.10.0' + + '@opentelemetry/sdk-node@0.217.0': + resolution: {integrity: sha512-K/60pSv42+NQiZKy1pAH18nYDkxltsDV4O3SJ233J0E9raU1ksyL9gsKuS8p30bYBb4AMPCfDuutHQaHYpcv0Q==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': '>=1.3.0 <1.10.0' @@ -4022,14 +4076,20 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/sdk-trace-base@2.7.1': + resolution: {integrity: sha512-NAYIlsF8MPUsKqJMiDQJTMPOmlbawC1Iz/omMLygZ1C9am8fTKYjTaI+OZM+WTY3t3Glo0wnOg/6/pac6RGPPw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/sdk-trace-node@2.10.0': resolution: {integrity: sha512-GZK/G6oZyBLGlH1pUgeDch7D91KoHd2uotUGIkWCPi9GI5T9X0p4L7nNAMDR1BQjkRYoDqo+ddfVx9t5Uhys+Q==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/sdk-trace-node@2.4.0': - resolution: {integrity: sha512-MBc2l04hZPYygnWPT38UiOPy9ueutPqmJ47z0m9IKuoVQh3MblmbSgwspjhdHagZLfSfmlzhWR1xtbgVNmjX2A==} + '@opentelemetry/sdk-trace-node@2.7.1': + resolution: {integrity: sha512-pCpQxU68lV+I9s9svqMyVu5iHdDDUnqUpSxqwyCU8A9ejEsSnMPCbearwsUO4yk08ZJzAIUCFuReMdVQvHrdvg==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' @@ -6410,7 +6470,7 @@ packages: engines: {node: '>= 22'} hasBin: true peerDependencies: - playwright-core: 1.61.1 + playwright-core: '*' caniuse-api@3.0.0: resolution: {integrity: sha512-bsTwuIg/BZZK/vreVTYYbSWoe2F+71P7K5QGEX+pT250DZbfU1MQ5prOKpPR+LL6uWKK3KMwMCAS74QB3Um1uw==} @@ -7609,6 +7669,9 @@ packages: es-module-lexer@2.0.0: resolution: {integrity: sha512-5POEcUuZybH7IdmGsD8wlf0AI55wMecM9rVBTI/qEAy2c1kTOm3DjFYjrBdI2K3BaJjJYfYFeRtM0t9ssnRuxw==} + es-module-lexer@2.3.2: + resolution: {integrity: sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==} + es-object-atoms@1.1.1: resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} engines: {node: '>= 0.4'} @@ -8767,6 +8830,10 @@ packages: import-in-the-middle@2.0.6: resolution: {integrity: sha512-3vZV3jX0XRFW3EJDTwzWoZa+RH1b8eTTx6YOCjglrLyPuepwoBti1k3L2dKwdCUrnVEfc5CuRuGstaC/uQJJaw==} + import-in-the-middle@3.4.0: + resolution: {integrity: sha512-Xfjwfarhe+LGmoaof+sexeNo3sGRysb5x56WrZLwHtmqT0OilSKtVWkv0lrCcMgSKyKP9oBogBAisHZvtJH0aw==} + engines: {node: '>=18'} + import-lazy@4.0.0: resolution: {integrity: sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==} engines: {node: '>=8'} @@ -10867,7 +10934,7 @@ packages: engines: {node: '>=12'} peerDependencies: playwright: '*' - playwright-core: 1.61.1 + playwright-core: '*' peerDependenciesMeta: playwright: optional: true @@ -11384,6 +11451,10 @@ packages: resolution: {integrity: sha512-jx6+sE9h/UryaCZhsJWbJtTEy47yXoGNYI4z8ZaRncM0zBKeRqjO2JEcOUYwrYGb1WLhXM1FfMzW3annvFv0rw==} engines: {node: '>=12.0.0'} + protobufjs@8.0.1: + resolution: {integrity: sha512-NWWCCscLjs+cOKF/s/XVNFRW7Yih0fdH+9brffR5NZCy8k42yRdl5KlWKMVXuI1vfCoy4o1z80XR/W/QUb3V3w==} + engines: {node: '>=12.0.0'} + protocols@2.0.2: resolution: {integrity: sha512-hHVTzba3wboROl0/aWRRG9dMytgH6ow//STBZh43l/wQgmMhYhOFi0EHWAPtoCz9IAUymsyP0TSBHkhgMEGNnQ==} @@ -14651,13 +14722,13 @@ snapshots: transitivePeerDependencies: - encoding - '@browserbasehq/stagehand@3.0.7(@cfworker/json-schema@4.1.1)(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(supports-color@8.1.1)(zod@4.4.3)': + '@browserbasehq/stagehand@3.0.7(@cfworker/json-schema@4.1.1)(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(supports-color@8.1.1)(zod@4.4.3)': dependencies: '@ai-sdk/provider': 2.0.1 '@anthropic-ai/sdk': 0.39.0(encoding@0.1.13) '@browserbasehq/sdk': 2.10.0(encoding@0.1.13) '@google/genai': 1.50.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(supports-color@7.2.0)(zod@4.4.3))(bufferutil@4.1.0)(supports-color@8.1.1) - '@langchain/openai': 0.4.9(@langchain/core@0.3.80(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)))(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0)) + '@langchain/openai': 0.4.9(@langchain/core@0.3.80(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)))(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0)) '@modelcontextprotocol/sdk': 1.29.0(@cfworker/json-schema@4.1.1)(supports-color@7.2.0)(zod@4.4.3) ai: 5.0.173(zod@4.4.3) deepmerge: 4.3.1 @@ -14684,7 +14755,7 @@ snapshots: '@ai-sdk/perplexity': 2.0.27(zod@4.4.3) '@ai-sdk/togetherai': 1.0.38(zod@4.4.3) '@ai-sdk/xai': 2.0.67(zod@4.4.3) - '@langchain/core': 0.3.80(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)) + '@langchain/core': 0.3.80(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)) bufferutil: 4.1.0 chrome-launcher: 1.2.1(supports-color@7.2.0) ollama-ai-provider-v2: 1.5.5(zod@4.4.3) @@ -16564,14 +16635,14 @@ snapshots: '@keyv/serialize@1.1.1': {} - '@langchain/core@0.3.80(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3))': + '@langchain/core@0.3.80(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3))': dependencies: '@cfworker/json-schema': 4.1.1 ansi-styles: 5.2.0 camelcase: 6.3.0 decamelize: 1.2.0 js-tiktoken: 1.0.21 - langsmith: 0.3.87(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)) + langsmith: 0.3.87(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)) mustache: 4.2.0 p-queue: 6.6.2 p-retry: 4.6.2 @@ -16584,9 +16655,9 @@ snapshots: - '@opentelemetry/sdk-trace-base' - openai - '@langchain/openai@0.4.9(@langchain/core@0.3.80(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)))(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))': + '@langchain/openai@0.4.9(@langchain/core@0.3.80(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)))(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))': dependencies: - '@langchain/core': 0.3.80(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)) + '@langchain/core': 0.3.80(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)) js-tiktoken: 1.0.21 openai: 4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@3.25.76) zod: 3.25.76 @@ -17048,19 +17119,23 @@ snapshots: dependencies: '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs@0.217.0': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api@1.9.0': {} - '@opentelemetry/configuration@0.210.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/configuration@0.217.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) yaml: 2.9.0 '@opentelemetry/context-async-hooks@2.10.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/context-async-hooks@2.4.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/context-async-hooks@2.7.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -17074,73 +17149,79 @@ snapshots: '@opentelemetry/api': 1.9.0 '@opentelemetry/semantic-conventions': 1.43.0 - '@opentelemetry/exporter-logs-otlp-grpc@0.210.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/core@2.7.1(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/exporter-logs-otlp-grpc@0.217.0(@opentelemetry/api@1.9.0)': dependencies: '@grpc/grpc-js': 1.14.4 '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-grpc-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-logs': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-grpc-exporter-base': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.217.0(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-logs-otlp-http@0.210.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/exporter-logs-otlp-http@0.217.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/api-logs': 0.210.0 - '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-logs': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/api-logs': 0.217.0 + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.217.0(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-logs-otlp-proto@0.210.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/exporter-logs-otlp-proto@0.217.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/api-logs': 0.210.0 - '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-logs': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/api-logs': 0.217.0 + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.7.1(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-metrics-otlp-grpc@0.210.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/exporter-metrics-otlp-grpc@0.217.0(@opentelemetry/api@1.9.0)': dependencies: '@grpc/grpc-js': 1.14.4 '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-metrics-otlp-http': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-grpc-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-metrics': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-metrics-otlp-http': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-grpc-exporter-base': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 2.7.1(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-metrics-otlp-http@0.210.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/exporter-metrics-otlp-http@0.217.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-metrics': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 2.7.1(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-metrics-otlp-proto@0.210.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/exporter-metrics-otlp-proto@0.217.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-metrics-otlp-http': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-metrics': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-metrics-otlp-http': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 2.7.1(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-prometheus@0.210.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/exporter-prometheus@0.217.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-metrics': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.43.0 '@opentelemetry/exporter-trace-otlp-grpc@0.210.0(@opentelemetry/api@1.9.0)': dependencies: @@ -17153,30 +17234,41 @@ snapshots: '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-trace-otlp-http@0.210.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/exporter-trace-otlp-grpc@0.217.0(@opentelemetry/api@1.9.0)': dependencies: + '@grpc/grpc-js': 1.14.4 '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-grpc-exporter-base': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.7.1(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/exporter-trace-otlp-http@0.217.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.7.1(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-zipkin@2.4.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.7.1(@opentelemetry/api@1.9.0) + + '@opentelemetry/exporter-zipkin@2.7.1(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.7.1(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.43.0 '@opentelemetry/instrumentation@0.210.0(@opentelemetry/api@1.9.0)(supports-color@7.2.0)': @@ -17197,12 +17289,27 @@ snapshots: transitivePeerDependencies: - supports-color + '@opentelemetry/instrumentation@0.217.0(@opentelemetry/api@1.9.0)(supports-color@8.1.1)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.217.0 + import-in-the-middle: 3.4.0 + require-in-the-middle: 8.0.1(supports-color@8.1.1) + transitivePeerDependencies: + - supports-color + '@opentelemetry/otlp-exporter-base@0.210.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base@0.217.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-grpc-exporter-base@0.210.0(@opentelemetry/api@1.9.0)': dependencies: '@grpc/grpc-js': 1.14.4 @@ -17211,6 +17318,14 @@ snapshots: '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-grpc-exporter-base@0.217.0(@opentelemetry/api@1.9.0)': + dependencies: + '@grpc/grpc-js': 1.14.4 + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer@0.210.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -17222,15 +17337,26 @@ snapshots: '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) protobufjs: 8.0.0 - '@opentelemetry/propagator-b3@2.4.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/otlp-transformer@0.217.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/api-logs': 0.217.0 + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.7.1(@opentelemetry/api@1.9.0) + protobufjs: 8.0.1 - '@opentelemetry/propagator-jaeger@2.4.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/propagator-b3@2.7.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + + '@opentelemetry/propagator-jaeger@2.7.1(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) '@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.0)': dependencies: @@ -17244,6 +17370,12 @@ snapshots: '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.43.0 + '@opentelemetry/resources@2.7.1(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.43.0 + '@opentelemetry/sdk-logs@0.210.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -17251,38 +17383,53 @@ snapshots: '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs@0.217.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.217.0 + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.43.0 + '@opentelemetry/sdk-metrics@2.4.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-node@0.210.0(@opentelemetry/api@1.9.0)(supports-color@8.1.1)': + '@opentelemetry/sdk-metrics@2.7.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/api-logs': 0.210.0 - '@opentelemetry/configuration': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/context-async-hooks': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-logs-otlp-grpc': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-logs-otlp-http': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-logs-otlp-proto': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-metrics-otlp-grpc': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-metrics-otlp-http': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-metrics-otlp-proto': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-prometheus': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-trace-otlp-grpc': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-trace-otlp-http': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-trace-otlp-proto': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-zipkin': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/instrumentation': 0.210.0(@opentelemetry/api@1.9.0)(supports-color@8.1.1) - '@opentelemetry/propagator-b3': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/propagator-jaeger': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-logs': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-metrics': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-trace-node': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.7.1(@opentelemetry/api@1.9.0) + + '@opentelemetry/sdk-node@0.217.0(@opentelemetry/api@1.9.0)(supports-color@8.1.1)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.217.0 + '@opentelemetry/configuration': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/context-async-hooks': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-logs-otlp-grpc': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-logs-otlp-http': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-logs-otlp-proto': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-metrics-otlp-grpc': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-metrics-otlp-http': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-metrics-otlp-proto': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-prometheus': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-trace-otlp-grpc': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-trace-otlp-http': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-trace-otlp-proto': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-zipkin': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/instrumentation': 0.217.0(@opentelemetry/api@1.9.0)(supports-color@8.1.1) + '@opentelemetry/otlp-exporter-base': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/propagator-b3': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/propagator-jaeger': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.217.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-node': 2.7.1(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.43.0 transitivePeerDependencies: - supports-color @@ -17302,6 +17449,13 @@ snapshots: '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.43.0 + '@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.43.0 + '@opentelemetry/sdk-trace-node@2.10.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -17309,12 +17463,12 @@ snapshots: '@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 2.10.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-trace-node@2.4.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/sdk-trace-node@2.7.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/context-async-hooks': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/context-async-hooks': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.7.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace@2.10.0(@opentelemetry/api@1.9.0)': dependencies: @@ -18992,9 +19146,9 @@ snapshots: optionalDependencies: ajv: 8.20.0 - ajv-formats@2.1.1(ajv@8.18.0): + ajv-formats@2.1.1(ajv@8.20.0): optionalDependencies: - ajv: 8.18.0 + ajv: 8.20.0 ajv-formats@3.0.1(ajv@8.18.0): optionalDependencies: @@ -19008,9 +19162,9 @@ snapshots: dependencies: ajv: 6.14.0 - ajv-keywords@5.1.0(ajv@8.18.0): + ajv-keywords@5.1.0(ajv@8.20.0): dependencies: - ajv: 8.18.0 + ajv: 8.20.0 fast-deep-equal: 3.1.3 ajv@6.14.0: @@ -21109,6 +21263,8 @@ snapshots: es-module-lexer@2.0.0: {} + es-module-lexer@2.3.2: {} + es-object-atoms@1.1.1: dependencies: es-errors: 1.3.0 @@ -22792,6 +22948,12 @@ snapshots: cjs-module-lexer: 2.2.1 module-details-from-path: 1.0.4 + import-in-the-middle@3.4.0: + dependencies: + cjs-module-lexer: 2.2.1 + es-module-lexer: 2.3.2 + module-details-from-path: 1.0.4 + import-lazy@4.0.0: {} import-local@3.1.0: @@ -23353,7 +23515,7 @@ snapshots: kuler@2.0.0: {} - langsmith@0.3.87(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)): + langsmith@0.3.87(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)): dependencies: '@types/uuid': 10.0.0 chalk: 4.1.2 @@ -23363,7 +23525,7 @@ snapshots: uuid: 10.0.0 optionalDependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/exporter-trace-otlp-proto': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-trace-otlp-proto': 0.217.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 2.10.0(@opentelemetry/api@1.9.0) openai: 4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3) @@ -26117,6 +26279,21 @@ snapshots: '@types/node': 24.12.2 long: 5.3.2 + protobufjs@8.0.1: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.5 + '@protobufjs/eventemitter': 1.1.1 + '@protobufjs/fetch': 1.1.1 + '@protobufjs/float': 1.0.2 + '@protobufjs/inquire': 1.1.2 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.2 + '@types/node': 24.12.2 + long: 5.3.2 + protocols@2.0.2: {} proxy-addr@2.0.7: @@ -26898,9 +27075,9 @@ snapshots: schema-utils@4.3.3: dependencies: '@types/json-schema': 7.0.15 - ajv: 8.18.0 - ajv-formats: 2.1.1(ajv@8.18.0) - ajv-keywords: 5.1.0(ajv@8.18.0) + ajv: 8.20.0 + ajv-formats: 2.1.1(ajv@8.20.0) + ajv-keywords: 5.1.0(ajv@8.20.0) search-insights@2.17.3: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index b8c36b54dce7..e7b054dff87b 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -16,16 +16,12 @@ minimumReleaseAgeExclude: - "got-scraping" overrides: - playwright-core: 1.61.1 - "@browserbasehq/stagehand": 3.0.7 # Dedup minimatch to v9 everywhere except inside lerna — lerna 9.x's bundled # code (`__toESM(require('minimatch')).default(...)`) only works with v3, # whose CJS export *is* the function. Pinning v9 there caused the publish # step to silently report 0 changed packages. minimatch: "^9.0.0" - tmp: "^0.2.7" "@puppeteer/browsers": "^3.0.4" - form-data: "^4.0.6" tar: "^7.5.16" "lerna>js-yaml": "^4.2.0" "lerna>minimatch": "^3.1.4" diff --git a/scripts/copy.ts b/scripts/copy.ts index 3ee28f625ab2..013bb16ae55d 100644 --- a/scripts/copy.ts +++ b/scripts/copy.ts @@ -98,6 +98,18 @@ function getNextVersion() { return `${version}-${preid}.${lastPrereleaseNumber + 1}`; } +// optionalDependencies too, otherwise `workspace:^` on @crawlee/impit-client publishes as a floating `^` range +function pinInternalDeps(pkgJson: any, version: string): void { + for (const deps of [pkgJson.dependencies, pkgJson.optionalDependencies]) { + for (const dep of Object.keys(deps ?? {})) { + if ((dep.startsWith('@crawlee/') && dep !== '@crawlee/fs-storage-native') || dep === 'crawlee') { + const prefix = deps[dep].startsWith('^') ? '^' : ''; + deps[dep] = prefix + version; + } + } + } +} + // as we publish only the dist folder, we need to copy some meta files inside (readme/license/package.json) // also changes paths inside the copied `package.json` (`dist/index.js` -> `index.js`) const root = resolve(import.meta.dirname, '..'); @@ -109,12 +121,7 @@ if (options.canary) { const nextVersion = getNextVersion(); pkgJson.version = nextVersion; - for (const dep of Object.keys(pkgJson.dependencies)) { - if ((dep.startsWith('@crawlee/') && dep !== '@crawlee/fs-storage-native') || dep === 'crawlee') { - const prefix = pkgJson.dependencies[dep].startsWith('^') ? '^' : ''; - pkgJson.dependencies[dep] = prefix + nextVersion; - } - } + pinInternalDeps(pkgJson, nextVersion); console.info(`canary: setting version to ${nextVersion}`); @@ -125,11 +132,7 @@ if (options['pin-versions']) { const pkgJson = require(pkgPath); const version = getRootVersion(false); - for (const dep of Object.keys(pkgJson.dependencies ?? {})) { - if ((dep.startsWith('@crawlee/') && dep !== '@crawlee/fs-storage-native') || dep === 'crawlee') { - pkgJson.dependencies[dep] = version; - } - } + pinInternalDeps(pkgJson, version); console.info(`pin-versions: version ${version}`, pkgJson.dependencies); diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts index cf6988f32eb1..86f1665c6a1c 100644 --- a/test/core/crawlers/basic_crawler.test.ts +++ b/test/core/crawlers/basic_crawler.test.ts @@ -339,8 +339,13 @@ describe('BasicCrawler', () => { requestHandler, }); + const queue = await basicCrawler.getRequestQueue(); + + // Nothing is emptied between runs, so the same sources are only re-crawled after a purge. await basicCrawler.run(sources); + await queue.purge?.(); await basicCrawler.run(sources); + await queue.purge?.(); await basicCrawler.run(sources); expect(processed).toHaveLength(sourcesCopy.length * 3); @@ -429,6 +434,63 @@ describe('BasicCrawler', () => { ]); }); + describe('a crawl that processes nothing', () => { + const crawlTwice = async (betweenRuns?: (crawler: BasicCrawler) => Promise) => { + const processed: string[] = []; + const crawler = new BasicCrawler({ + requestHandler: async ({ request }) => { + processed.push(request.url); + }, + }); + const warning = vitest.spyOn(crawler.log, 'warning'); + + await crawler.run(['https://example.com/only']); + await betweenRuns?.(crawler); + await crawler.run(['https://example.com/only']); + + return { processed, warning }; + }; + + test('leaves already handled requests alone and says so', async () => { + const { processed, warning } = await crawlTwice(); + + expect(processed).toEqual(['https://example.com/only']); + expect(warning).toHaveBeenCalledWith(expect.stringMatching(/processed no requests/)); + }); + + test('re-crawls the requests when the queue is purged in between', async () => { + const { processed, warning } = await crawlTwice(async (crawler) => { + await (await crawler.getRequestQueue()).purge?.(); + }); + + expect(processed).toEqual(['https://example.com/only', 'https://example.com/only']); + expect(warning).not.toHaveBeenCalledWith(expect.stringMatching(/processed no requests/)); + }); + + test('warns a crawler on its first run, over a queue another crawler exhausted', async () => { + const requestQueue = await RequestQueue.open(); + const first = new BasicCrawler({ requestQueue, requestHandler: async () => {} }); + await first.run(['https://example.com/only']); + + const second = new BasicCrawler({ requestQueue, requestHandler: async () => {} }); + const warning = vitest.spyOn(second.log, 'warning'); + + await second.run(['https://example.com/only']); + + expect(warning).toHaveBeenCalledWith(expect.stringMatching(/processed no requests/)); + }); + + test('says nothing when there was nothing to crawl in the first place', async () => { + // An empty queue is not evidence of a mistake - only requests that turn out to be handled are. + const crawler = new BasicCrawler({ requestHandler: async () => {} }); + const warning = vitest.spyOn(crawler.log, 'warning'); + + await crawler.run(); + + expect(warning).not.toHaveBeenCalledWith(expect.stringMatching(/processed no requests/)); + }); + }); + test('addRequests should respect maxCrawlDepth', async () => { const processedUrls: string[] = []; @@ -2880,107 +2942,8 @@ describe('BasicCrawler', () => { expect(visits[1].at - visits[0].at).toBeGreaterThanOrEqual(400); }); - test('a second run() crawls the same requests again', async () => { - const crawler = new BasicCrawler({ - sameDomainDelaySecs: 0.05, - requestHandler: async () => {}, - }); - - await crawler.run(['http://example.com/1']); - await crawler.run(['http://example.com/1']); - - expect(crawler.statistics.state.requestsFinished).toBe(1); - }); - - test('a second run() over a supplied manager asks which storage to purge', async () => { - // A purge cannot respect both the crawler's per-domain queues and the caller's manager - // underneath them, so rather than pick one silently, say so. - const crawler = new BasicCrawler({ - requestQueue: await RequestQueue.open(), - sameDomainDelaySecs: 0.05, - requestHandler: async () => {}, - }); - - await crawler.run(['http://example.com/1']); - - await expect(crawler.run(['http://example.com/1'])).rejects.toThrow( - /Cannot decide what to purge.*purgeRequestQueue/s, - ); - }); - - test.each([true, false])('a second run() with purgeRequestQueue: %s does not ask', async (purge) => { - const crawler = new BasicCrawler({ - requestQueue: await RequestQueue.open(), - sameDomainDelaySecs: 0.05, - requestHandler: async () => {}, - }); - - await crawler.run(['http://example.com/1']); - - await expect( - crawler.run(['http://example.com/1'], { purgeRequestQueue: purge }), - ).resolves.toBeDefined(); - }); - - test('a second run() asks even when the first routed nothing by domain', async () => { - // The guard fires on the shape of the configuration, not on what the queues happen to hold: - // keying it on an existing per-domain queue would make identical code throw or not depending - // on run history. - const requestQueue = await RequestQueue.open(); - await requestQueue.addRequest({ url: 'http://example.com/pre-added-1' }); - await requestQueue.addRequest({ url: 'http://example.com/pre-added-2' }); - - const visits: number[] = []; - const crawler = new BasicCrawler({ - requestQueue, - sameDomainDelaySecs: 2, - requestHandler: async () => { - visits.push(Date.now()); - }, - }); - - await crawler.run(); - - // Precondition, established rather than assumed: no per-domain queue was opened, since one - // would have paced these 2s apart. - expect(visits).toHaveLength(2); - expect(visits[1] - visits[0]).toBeLessThan(1000); - - await expect(crawler.run()).rejects.toThrow(/Cannot decide what to purge/); - }); - - test('a second run() asks for a supplied `requestManager`, not just a `requestQueue`', async () => { - const crawler = new BasicCrawler({ - requestManager: await RequestQueue.open(), - sameDomainDelaySecs: 0.05, - requestHandler: async () => {}, - }); - - await crawler.run(['http://example.com/1']); - - await expect(crawler.run(['http://example.com/1'])).rejects.toThrow(/Cannot decide what to purge/); - }); - - test('a second run() leaves a supplied manager alone when nothing paces it', async () => { - // The contrast that makes the question above worth asking: with no `sameDomainDelaySecs` the - // crawler puts nothing of its own inside the caller's manager, so there is nothing to ask about. - let visits = 0; - const crawler = new BasicCrawler({ - requestQueue: await RequestQueue.open(), - requestHandler: async () => { - visits += 1; - }, - }); - - await crawler.run(['http://example.com/1']); - await crawler.run(['http://example.com/1']); - - expect(visits).toBe(1); - }); - - test('a second run() purges everything the crawler opened itself, without being asked', async () => { - // Nothing came from the caller, so there is nothing to ask about - and the purge has to reach - // the per-domain queues, or the second run would crawl nothing. + test('a second run() does not crawl the same requests again', async () => { + // The per-domain queues the pacer opened are not emptied between runs either. let visits = 0; const crawler = new BasicCrawler({ sameDomainDelaySecs: 0.05, @@ -2992,7 +2955,7 @@ describe('BasicCrawler', () => { await crawler.run(['http://example.com/1']); await crawler.run(['http://example.com/1']); - expect(visits).toBe(2); + expect(visits).toBe(1); }); test('wraps a user `requestManager` rather than replacing it', async () => { @@ -3068,27 +3031,6 @@ describe('BasicCrawler', () => { }), ).toThrow(/domains: 'all'/); }); - - test('a second run() leaves a manager that took the delay alone', async () => { - // The floor put no storage of ours underneath it, so there is no purge to ask about. - let visits = 0; - const crawler = new BasicCrawler({ - requestManager: new ThrottlingRequestManager({ - inner: await RequestQueue.open(), - domains: 'all', - throttleBy: 'registrableDomain', - }), - sameDomainDelaySecs: 0.05, - requestHandler: async () => { - visits += 1; - }, - }); - - await crawler.run(['http://example.com/1']); - await crawler.run(['http://example.com/1']); - - expect(visits).toBe(1); - }); }); test('enqueueLinks should respect custom user-agent robots.txt rules', async () => { @@ -3721,6 +3663,53 @@ describe('BasicCrawler', () => { await expect(KeyValueStore.getValue('skipped')).resolves.toMatchObject({ reason: 'robotsTxt' }); }); + test('a crawler started from inside a request handler runs outside the caller transaction', async () => { + const handled: string[] = []; + + const inner = new BasicCrawler({ + requestHandler: async ({ request }) => { + handled.push(request.url); + }, + }); + + const outer = new BasicCrawler({ + maxRequestRetries: 0, + requestHandler: async () => { + const run = inner.run([`http://${HOSTNAME}:${port}/inner`]); + await new Promise((resolve) => setTimeout(resolve, 200)); + await run; + }, + }); + + await outer.run([`http://${HOSTNAME}:${port}/outer`]); + + expect(handled).toEqual([`http://${HOSTNAME}:${port}/inner`]); + }); + + test('a nested crawl is not cancelled by the calling request handler timing out', async () => { + const handled: string[] = []; + + const inner = new BasicCrawler({ + maxRequestRetries: 0, + requestHandler: async ({ request }) => { + await new Promise((resolve) => setTimeout(resolve, 1500)); + handled.push(request.url); + }, + }); + + const outer = new BasicCrawler({ + maxRequestRetries: 0, + requestHandlerTimeoutSecs: 1, + requestHandler: async () => { + void inner.run([`http://${HOSTNAME}:${port}/inner`]); + await new Promise((resolve) => setTimeout(resolve, 5000)); + }, + }); + + await outer.run([`http://${HOSTNAME}:${port}/outer`]); + await expect.poll(() => handled).toEqual([`http://${HOSTNAME}:${port}/inner`]); + }); + test('an unrecognized write policy is rejected instead of falling back to the default', () => { const make = (transactionalStorage: unknown) => new BasicCrawler({ requestHandler: async () => {}, transactionalStorage } as any); diff --git a/test/core/storages/dataset.test.ts b/test/core/storages/dataset.test.ts index a210088b6368..f86c34870b4b 100644 --- a/test/core/storages/dataset.test.ts +++ b/test/core/storages/dataset.test.ts @@ -719,3 +719,17 @@ describe('dataset', () => { }); }); }); + +describe('Dataset.purge', () => { + test('empties the dataset but keeps it usable', async () => { + const dataset = await Dataset.open(); + await dataset.pushData([{ n: 1 }, { n: 2 }]); + + await dataset.purge(); + + await expect(dataset.getData()).resolves.toMatchObject({ items: [], total: 0 }); + + await dataset.pushData({ n: 3 }); + await expect(dataset.getData()).resolves.toMatchObject({ items: [{ n: 3 }] }); + }); +}); diff --git a/test/core/storages/key_value_store.test.ts b/test/core/storages/key_value_store.test.ts index fff61bdfe3ee..7090dc00534d 100644 --- a/test/core/storages/key_value_store.test.ts +++ b/test/core/storages/key_value_store.test.ts @@ -721,3 +721,22 @@ describe('KeyValueStore', () => { }); }); }); + +describe('KeyValueStore.purge', () => { + test('empties the store but keeps it usable, and forgets auto-saved values', async () => { + const store = await KeyValueStore.open(); + await store.setValue('key-1', { foo: 'bar' }); + const state = await store.getAutoSavedValue('STATE', { hits: 0 }); + state.hits = 1; + + await store.purge(); + + await expect(store.getValue('key-1')).resolves.toBeNull(); + + // The auto-save cache would otherwise keep serving a value the store no longer holds. + await expect(store.getAutoSavedValue('STATE', { hits: 0 })).resolves.toEqual({ hits: 0 }); + + await store.setValue('key-2', { foo: 'baz' }); + await expect(store.getValue('key-2')).resolves.toEqual({ foo: 'baz' }); + }); +}); diff --git a/test/core/storages/storage_transaction.test.ts b/test/core/storages/storage_transaction.test.ts index 87fdd6e23fdf..945781575ee9 100644 --- a/test/core/storages/storage_transaction.test.ts +++ b/test/core/storages/storage_transaction.test.ts @@ -344,11 +344,12 @@ describe('Dataset in a transaction', () => { expect(pushDataSpy).toHaveBeenCalledWith([{ n: 1 }, { n: 2 }, { n: 3 }]); }); - test('drop is rejected inside a transaction and allowed under withDirectStorageAccess', async () => { + test('drop and purge are rejected inside a transaction and allowed under withDirectStorageAccess', async () => { const dataset = await Dataset.open(); await withStorageTransaction(async () => { await expect(dataset.drop()).rejects.toThrow(/cannot be used inside a storage transaction/); + await expect(dataset.purge()).rejects.toThrow(/cannot be used inside a storage transaction/); await withDirectStorageAccess(async () => dataset.drop()); }); }); @@ -588,11 +589,12 @@ describe('KeyValueStore in a transaction', () => { await expect(store.getValue('COMMITTED_STATE')).resolves.toEqual({ a: 2 }); }); - test('drop and clearCache are rejected inside a transaction', async () => { + test('drop, purge and clearCache are rejected inside a transaction', async () => { const store = await KeyValueStore.open(); await withStorageTransaction(async () => { await expect(store.drop()).rejects.toThrow(/cannot be used inside a storage transaction/); + await expect(store.purge()).rejects.toThrow(/cannot be used inside a storage transaction/); expect(() => store.clearCache()).toThrow(/cannot be used inside a storage transaction/); }); }); diff --git a/test/e2e/cheerio-stop-resume-ts/actor/main.ts b/test/e2e/cheerio-stop-resume-ts/actor/main.ts index 737ea8cb649c..028e140a2187 100644 --- a/test/e2e/cheerio-stop-resume-ts/actor/main.ts +++ b/test/e2e/cheerio-stop-resume-ts/actor/main.ts @@ -27,5 +27,5 @@ crawler.router.addDefaultHandler(async ({ $, enqueueLinks, request, log }) => { await crawler.run(['https://crawlee.dev/js/docs/quick-start']); requestCount = 0; -await crawler.run(['https://crawlee.dev/js/docs/quick-start'], { purgeRequestQueue: false }); +await crawler.run(['https://crawlee.dev/js/docs/quick-start']); await Actor.exit({ exit: Actor.isAtHome() }); diff --git a/test/e2e/cheerio-stop-resume-ts/test.mjs b/test/e2e/cheerio-stop-resume-ts/test.mjs index 8beaf8681c80..6611a0a4756d 100644 --- a/test/e2e/cheerio-stop-resume-ts/test.mjs +++ b/test/e2e/cheerio-stop-resume-ts/test.mjs @@ -9,4 +9,4 @@ const { stats, datasetItems } = await runActor(testActorDirname); await expect(stats.requestsFinished < 40, 'crawler.stop() works'); const visitedUrls = new Set(datasetItems.map((x) => x.url)); -await expect(visitedUrls.size === datasetItems.length, 'stateful crawler.run({ purgeRQ: false }) works'); +await expect(visitedUrls.size === datasetItems.length, 'a second crawler.run() resumes the same queue');