diff --git a/docs/guides/running-in-web-server/running-in-web-server.mdx b/docs/guides/running-in-web-server/running-in-web-server.mdx index f12038e0f5d2..c31bc0a880d4 100644 --- a/docs/guides/running-in-web-server/running-in-web-server.mdx +++ b/docs/guides/running-in-web-server/running-in-web-server.mdx @@ -58,3 +58,9 @@ Now we need to glue the server and the crawler together using the mapping of Cra {WebServerSource} +## The request queue keeps growing + +Nothing empties a request queue on its own. Every incoming HTTP request gets its own `uniqueKey`, so none of them deduplicate either. The queue therefore holds one handled request for every request the server has served. The records are small, and on the Apify platform they live server-side. It only matters for a server staying up a very long time, where the answer is to recycle the process. + +Do not reach for `purge()` while the crawler is running. It empties the whole queue, including requests added but not yet picked up. Every HTTP request still waiting is dropped and never answered, and its entry in `requestsToResponses` leaks. The more `maxConcurrency` throttles the crawler, the more requests are sitting there to lose. + diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md index e151e40c28ec..1d4a1ba96d43 100644 --- a/docs/upgrading/upgrading_v4.md +++ b/docs/upgrading/upgrading_v4.md @@ -615,17 +615,33 @@ The `purgeRequestQueue` option of `crawler.run()` went away with the automatic p const crawler = new BasicCrawler({ requestHandler: async ({ request }) => { /* ... */ } }); await crawler.run(['https://example.com/a', 'https://example.com/b']); -const queue = await crawler.getRequestQueue(); -await queue.purge?.(); +const requestManager = await crawler.getRequestManager(); +await requestManager.purge?.(); // The same URLs are crawled again: await crawler.run(['https://example.com/a', 'https://example.com/c']); ``` -`purge()` — empty the storage, keep its id and name — is new in v4 and available on `Dataset`, `KeyValueStore` and `RequestQueue`, as well as being an optional method on the `IRequestManager` interface. +`purge()` — empty the storage, keep its id and name — is new in v4 and available on `Dataset`, `KeyValueStore` and `RequestQueue`, as well as being an optional method on the `IRequestManager` interface. The Apify platform is the exception. Its API has no in-place empty, so all three throw there. The error points you at `drop()` or a fresh storage. This has nothing to do with `purgeOnStart` / `CRAWLEE_PURGE_ON_START`, which still wipes the default storages once per process before the first run. +Most of the time you can avoid the purge entirely. Every crawler instance opens a request queue of its own (see the section above). A crawler per crawl therefore needs neither a purge nor any queue wiring. Pass `RequestQueue.open({ alias })` when you do want to hold on to that queue: + +```typescript +for (const [index, urls] of batches.entries()) { + const crawler = new BasicCrawler({ + // Optional — a fresh crawler gets its own queue anyway. Pass one to decide which. + requestManager: await RequestQueue.open({ alias: `batch-${index}` }), + requestHandler: async ({ request }) => { /* ... */ }, + }); + + await crawler.run(urls); +} +``` + +An alias identifies a run-scoped queue. It has no persistent name, and is emptied on start along with the default storages. Reuse an alias and you get that same queue back, handled requests included. The next crawl then finds nothing to do. Give each crawl its own alias. `purge()` is for when one crawler and one queue must be reused. + ### Storage `.open()` now also accepts `{ id?, name? }` `Dataset.open()`, `KeyValueStore.open()`, and `RequestQueue.open()` previously accepted a single `idOrName?: string` parameter. This was ambiguous — callers couldn't express whether they were opening a storage by its ID or by name. diff --git a/test/e2e/playwright-multi-run/actor/.actor/actor.json b/test/e2e/playwright-multi-run/actor/.actor/actor.json deleted file mode 100644 index 28976780d31a..000000000000 --- a/test/e2e/playwright-multi-run/actor/.actor/actor.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "actorSpecification": 1, - "name": "test-playwright-multi-run", - "version": "0.0", - "buildTag": "latest", - "env": null -} diff --git a/test/e2e/playwright-multi-run/actor/.gitignore b/test/e2e/playwright-multi-run/actor/.gitignore deleted file mode 100644 index ced7cbfc582d..000000000000 --- a/test/e2e/playwright-multi-run/actor/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -.idea -.DS_Store -node_modules -package-lock.json -apify_storage -crawlee_storage -storage diff --git a/test/e2e/playwright-multi-run/actor/Dockerfile b/test/e2e/playwright-multi-run/actor/Dockerfile deleted file mode 100644 index 77da7b583c88..000000000000 --- a/test/e2e/playwright-multi-run/actor/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -FROM node:24 AS builder - -COPY /packages ./packages -COPY /package*.json ./ -RUN rm -rf node_modules package-lock.json -RUN npm --quiet set progress=false \ - && npm install --only=prod --no-audit \ - && npm update - -FROM apify/actor-node-playwright-chrome:24-1.58.2-beta - -# The browser base images ship a preinstalled node_modules - wipe it so the -# builder tree (with file: symlinks) can be copied over cleanly. -RUN rm -rf node_modules -COPY --from=builder /node_modules ./node_modules -COPY --from=builder /packages ./packages -COPY --from=builder /package*.json ./ -COPY /.actor ./.actor -COPY /main.js ./ - -RUN echo "Installed NPM packages:" \ - && (npm list --only=prod --all || true) \ - && echo "Node.js version:" \ - && node --version \ - && echo "NPM version:" \ - && npm --version diff --git a/test/e2e/playwright-multi-run/actor/main.js b/test/e2e/playwright-multi-run/actor/main.js deleted file mode 100644 index 21191bdbba1f..000000000000 --- a/test/e2e/playwright-multi-run/actor/main.js +++ /dev/null @@ -1,30 +0,0 @@ -import { Actor } from 'apify'; -import { Dataset, PlaywrightCrawler } from '@crawlee/playwright'; - -if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') { - await Actor.init({ storage: new (await import('@apify/storage-local')).ApifyStorageLocal() }); -} else { - await Actor.init(); -} - -const crawler = new PlaywrightCrawler({ - maxRequestsPerCrawl: 10, - async requestHandler({ log, page, enqueueLinks, request }) { - const { url } = request; - log.info(`Processing ${url}...`); - const pageTitle = await page.title(); - await Dataset.pushData({ url, pageTitle }); - await enqueueLinks({ - include: ['**/3.12/examples/*'], - }); - }, -}); - -crawler.log.info('=== Run 1 ==='); -await crawler.run(['https://crawlee.dev/js/docs/3.12/examples']); -crawler.log.info('=== Run 2 ==='); -await crawler.run(['https://crawlee.dev/js/docs/3.12/examples']); -crawler.log.info('=== Run 3 ==='); -await crawler.run(['https://crawlee.dev/js/docs/3.12/examples']); - -await Actor.exit({ exit: Actor.isAtHome() }); diff --git a/test/e2e/playwright-multi-run/actor/package.json b/test/e2e/playwright-multi-run/actor/package.json deleted file mode 100644 index 376373749818..000000000000 --- a/test/e2e/playwright-multi-run/actor/package.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "test-playwright-multi-run", - "version": "0.0.1", - "description": "Playwright Test - Multiple run calls to the same crawler", - "dependencies": { - "apify": "^4.0.0-beta.24", - "@crawlee/basic": "file:./packages/basic-crawler", - "@crawlee/browser": "file:./packages/browser-crawler", - "@crawlee/browser-pool": "file:./packages/browser-pool", - "@crawlee/core": "file:./packages/core", - "@crawlee/playwright": "file:./packages/playwright-crawler", - "@crawlee/types": "file:./packages/types", - "@crawlee/utils": "file:./packages/utils", - "playwright": "1.58.2" - }, - "overrides": { - "apify": { - "@crawlee/core": "file:./packages/core", - "@crawlee/utils": "file:./packages/utils" - } - }, - "scripts": { - "start": "node main.js" - }, - "type": "module", - "license": "ISC" -} diff --git a/test/e2e/playwright-multi-run/test.mjs b/test/e2e/playwright-multi-run/test.mjs deleted file mode 100644 index 55e1c47b05d0..000000000000 --- a/test/e2e/playwright-multi-run/test.mjs +++ /dev/null @@ -1,14 +0,0 @@ -import { expect, getActorTestDir, initialize, runActor, skipTest, validateDataset } from '../tools.mjs'; - -if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') { - await skipTest('not supported on platform'); -} - -const testActorDirname = getActorTestDir(import.meta.url); -await initialize(testActorDirname); - -const { datasetItems } = await runActor(testActorDirname, 16384); - -// we cant assert number of requests as the stats KVS is being wiped on each `run` call, unlike the dataset -await expect(datasetItems.length >= 30, `Number of dataset items (actual: ${datasetItems.length}, expected: >= 30)`); -await expect(validateDataset(datasetItems, ['url', 'pageTitle']), 'Dataset items validation');