[pull] master from apify:master

docs/public-api/crawlee-core.api.md

@@ -1797,7 +1797,7 @@ export interface SitemapRequestLoaderOptions extends UrlConstraints {
 
 // @public (undocumented)
 export type SkippedRequestCallback = (args: {
-    url: string;
+    request: Request_2;
     reason: SkippedRequestReason;
 }) => Awaitable<void>;
 

docs/upgrading/upgrading_v4.md

@@ -843,6 +843,10 @@ const urls = await extractLinks({ selector: '.product-link' });
 
 The `robotsTxtFile` / `respectRobotsTxtFile` per-call options are removed from `enqueueLinks()` — robots.txt filtering is applied by the crawler consistently via `BasicCrawlerOptions.respectRobotsTxtFile`.
 
+### `onSkippedRequest` receives a `Request` instead of a URL string
+
+The callback now gets `{ request, reason }` instead of `{ url, reason }` — use `request.url` for the URL.
+
 ### Internal KVS keys renamed
 
 Several internal Crawlee keys were prefixed with the `SDK_` prefix for legacy reasons — these keys now start with `CRAWLEE_` instead. These are, e.g., `CRAWLEE_SESSION_POOL_STATE` or `CRAWLEE_CRAWLER_STATISTICS_{n}`.

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -18,6 +18,7 @@ import type {
     IRequestLoader,
     IRequestManager,
     IStatistics,
+    RequestOptions,
     RequestsLike,
     RouterHandler,
     RouterRoutes,
@@ -45,6 +46,7 @@ import {
     ContextPipelineInitializationError,
     ContextPipelineInterruptedError,
     createRequestOptions,
+    createSkippedRequestArgs,
     createStorageTransaction,
     Request,
     CriticalError,
@@ -1454,7 +1456,7 @@ export class BasicCrawler<
             request.state = RequestState.SKIPPED;
             request.noRetry = true;
             await this.#handleSkippedRequest({
-                url: request.url,
+                request,
                 reason: 'robotsTxt',
             });
 
@@ -1587,7 +1589,7 @@ export class BasicCrawler<
                     request.noRetry = true;
                     request.state = RequestState.SKIPPED;
 
-                    await this.#handleSkippedRequest({ url: request.url, reason: 'redirect' });
+                    await this.#handleSkippedRequest({ request, reason: 'redirect' });
 
                     throw new ContextPipelineInterruptedError(message);
                 }
@@ -2099,7 +2101,12 @@ export class BasicCrawler<
         const maxCrawlDepth = this.#maxCrawlDepth;
         const validateRequestUserData = this.validateRequestUserData.bind(this);
 
-        const allSkipped: { url: string; reason: SkippedRequestReason }[] = [];
+        const allSkipped: { source: string | Source; reason: SkippedRequestReason }[] = [];
+        // A skipped source (which can carry arbitrary userData) is only retained if something reads it -
+        // otherwise the URL alone is enough to build the callback argument and to log with.
+        const hasSkippedRequestCallback =
+            this.#onSkippedRequest !== undefined || options.onSkippedRequest !== undefined;
+        const keepSkippedSource = (source: Source) => (hasSkippedRequestCallback ? source : source.url!);
 
         async function* filteredRequests() {
             for await (const request of requests) {
@@ -2113,32 +2120,33 @@ export class BasicCrawler<
                 }
 
                 if (maxCrawlDepth !== undefined && requestOptions.crawlDepth! > maxCrawlDepth) {
-                    allSkipped.push({ url: requestOptions.url, reason: 'depth' });
+                    allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'depth' });
                     continue;
                 }
 
                 if (!(await isAllowedBasedOnRobotsTxtFile(requestOptions.url))) {
-                    allSkipped.push({ url: requestOptions.url, reason: 'robotsTxt' });
+                    allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'robotsTxt' });
                     continue;
                 }
 
-                const onSkippedFilterUrl = (url: string) => allSkipped.push({ url, reason: 'filters' });
+                const onSkippedByFilter = (opts: RequestOptions) =>
+                    allSkipped.push({ source: keepSkippedSource(opts), reason: 'filters' });
 
                 // Filter by user patterns first (with exclude)...
                 let filtered = filterRequestOptionsByPatterns(
                     [requestOptions],
                     urlPatternObjects.length > 0 ? urlPatternObjects : undefined,
                     urlExcludePatternObjects,
                     strategy,
-                    onSkippedFilterUrl,
+                    onSkippedByFilter,
                 );
                 // ...then filter by the enqueue strategy (making this an AND check)
                 filtered = filterRequestOptionsByPatterns(
                     filtered,
                     enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined,
                     [],
                     strategy,
-                    onSkippedFilterUrl,
+                    onSkippedByFilter,
                 );
 
                 if (filtered.length === 0) {
@@ -2149,7 +2157,7 @@ export class BasicCrawler<
 
                 if (options.transformRequestFunction) {
                     const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) =>
-                        allSkipped.push({ url: r.url, reason: r.skippedReason ?? 'transform' }),
+                        allSkipped.push({ source: keepSkippedSource(r), reason: r.skippedReason ?? 'transform' }),
                     );
 
                     if (transformed.length === 0) {
@@ -2175,11 +2183,13 @@ export class BasicCrawler<
         // Report requests skipped due to the maxNewRequests budget (i.e. maxRequestsPerCrawl limit, or an
         // explicit `limit` option)
         for (const request of result.requestsOverLimit ?? []) {
-            allSkipped.push({ url: typeof request === 'string' ? request : request.url!, reason: 'limit' });
+            allSkipped.push({ source: request, reason: 'limit' });
         }
 
         if (allSkipped.length > 0) {
-            const skippedRobotsUrls = allSkipped.filter((s) => s.reason === 'robotsTxt').map((s) => s.url);
+            const skippedRobotsUrls = allSkipped
+                .filter((s) => s.reason === 'robotsTxt')
+                .map(({ source }) => (typeof source === 'string' ? source : source.url!));
             if (skippedRobotsUrls.length > 0) {
                 this.log.warning(
                     `Some requests were skipped because they were disallowed based on the robots.txt file`,
@@ -2198,9 +2208,10 @@ export class BasicCrawler<
             }
 
             await Promise.all(
-                allSkipped.map(async ({ url, reason }) => {
-                    await this.#handleSkippedRequest({ url, reason });
-                    await options.onSkippedRequest?.({ url, reason });
+                allSkipped.map(async ({ source, reason }) => {
+                    const args = createSkippedRequestArgs(source, reason);
+                    await this.#handleSkippedRequest(args);
+                    await options.onSkippedRequest?.(args);
                 }),
             );
         }

packages/core/src/enqueue_links/enqueue_links.ts

@@ -179,8 +179,7 @@ export function resolveBaseUrlForEnqueueLinksFiltering({
     }
 
     // If the user wants to ensure the same domain is accessed, regardless of subdomains, we check to ensure the domains match
-    // Returning undefined here is intentional! If the domains don't match, having no baseUrl in enqueueLinks will cause it to not enqueue anything
-    // which is the intended behavior (since we went off domain)
+    // If they don't (we went off domain via a redirect), we keep filtering against the original domain - returning undefined would disable the filtering entirely
     if (enqueueStrategy === EnqueueStrategy.SameDomain) {
         const originalHostname = getDomain(originalUrlOrigin, { mixedInputs: false })!;
         const finalHostname = getDomain(finalUrlOrigin, { mixedInputs: false })!;
@@ -189,7 +188,7 @@ export function resolveBaseUrlForEnqueueLinksFiltering({
             return finalUrlOrigin;
         }
 
-        return undefined;
+        return originalUrlOrigin;
     }
 
     // Always enqueue urls that are from the same origin in all other cases, as the filtering happens on the original request url, even if there was a redirect

packages/core/src/enqueue_links/shared.ts

@@ -4,7 +4,8 @@ import type { Awaitable, Dictionary } from '@crawlee/types';
 import { Minimatch } from 'minimatch';
 import { z } from 'zod';
 
-import type { RequestOptions } from '../request.js';
+import type { RequestOptions, Source } from '../request.js';
+import { Request } from '../request.js';
 import { schemas } from '../validators.js';
 import type { EnqueueStrategyOption } from './enqueue_links.js';
 
@@ -57,7 +58,32 @@ export type SkippedRequestReason =
     | 'redirect'
     | 'depth';
 
-export type SkippedRequestCallback = (args: { url: string; reason: SkippedRequestReason }) => Awaitable<void>;
+export type SkippedRequestCallback = (args: { request: Request; reason: SkippedRequestReason }) => Awaitable<void>;
+
+/**
+ * Builds the `{ request, reason }` argument passed to a {@apilink SkippedRequestCallback}, constructing the
+ * `Request` lazily on first access to `request` (and caching it) since most skips are never observed by a
+ * real callback and building a `Request` isn't free.
+ * @ignore
+ */
+export function createSkippedRequestArgs(
+    source: string | Source,
+    reason: SkippedRequestReason,
+): Parameters<SkippedRequestCallback>[0] {
+    const sourceReason = typeof source === 'string' ? undefined : source.skippedReason;
+    let request: Request | undefined;
+
+    return {
+        reason: sourceReason ?? reason,
+        get request() {
+            request ??=
+                source instanceof Request
+                    ? source
+                    : new Request(typeof source === 'string' ? { url: source } : (source as RequestOptions));
+            return request;
+        },
+    };
+}
 
 /**
  * @ignore
@@ -176,16 +202,16 @@ export function filterRequestOptionsByPatterns(
     includePatterns: UrlPatternObject[] | undefined,
     excludePatterns: UrlPatternObject[] = [],
     strategy?: EnqueueStrategyOption,
-    onSkippedUrl?: (url: string) => void,
+    onSkippedRequestOptions?: (options: RequestOptions) => void,
 ): RequestOptions[] {
     const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
     const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
 
     return requestOptions
-        .filter(({ url }) => {
-            const matchesExclude = excludeMatchers.some(({ match }) => match(url));
+        .filter((opts) => {
+            const matchesExclude = excludeMatchers.some(({ match }) => match(opts.url));
             if (matchesExclude) {
-                onSkippedUrl?.(url);
+                onSkippedRequestOptions?.(opts);
             }
             return !matchesExclude;
         })
@@ -201,7 +227,7 @@ export function filterRequestOptionsByPatterns(
             }
 
             // didn't match any positive pattern
-            onSkippedUrl?.(opts.url);
+            onSkippedRequestOptions?.(opts);
             return null;
         })
         .filter((opts) => opts !== null);

packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts

@@ -12,6 +12,7 @@ import {
     applyRequestTransform,
     constructUrlPatternObjects,
     createRequestOptions,
+    createSkippedRequestArgs,
     filterRequestOptionsByPatterns,
     urlPatternSchema,
     Request as CrawleeRequest,
@@ -236,6 +237,7 @@ export async function enqueueLinksByClickingElements(
 
     const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
     const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
+    const hasOnSkippedRequest = onSkippedRequest !== undefined;
 
     const urlExcludePatternObjects: UrlPatternObject[] = exclude?.length ? constructUrlPatternObjects(exclude) : [];
     const urlPatternObjects: UrlPatternObject[] = include?.length ? constructUrlPatternObjects(include) : [];
@@ -248,27 +250,31 @@ export async function enqueueLinksByClickingElements(
         clickOptions,
     });
     const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);
-    const skippedByFilters: string[] = [];
+    const skippedByFilters: RequestOptions[] = [];
     let filteredOptions = filterRequestOptionsByPatterns(
         requestOptions,
         urlPatternObjects.length > 0 ? urlPatternObjects : undefined,
         urlExcludePatternObjects,
         undefined,
-        (url) => skippedByFilters.push(url),
+        hasOnSkippedRequest ? (opts) => skippedByFilters.push(opts) : undefined,
     );
 
     if (onSkippedRequest && skippedByFilters.length > 0) {
-        await Promise.all(skippedByFilters.map(async (url) => onSkippedRequest({ url, reason: 'filters' })));
+        await Promise.all(
+            skippedByFilters.map(async (opts) => onSkippedRequest(createSkippedRequestArgs(opts, 'filters'))),
+        );
     }
 
     if (transformRequestFunction) {
         const skippedByTransform: RequestOptions[] = [];
-        filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) =>
-            skippedByTransform.push(r),
+        filteredOptions = applyRequestTransform(
+            filteredOptions,
+            transformRequestFunction,
+            hasOnSkippedRequest ? (r) => skippedByTransform.push(r) : undefined,
         );
         if (onSkippedRequest && skippedByTransform.length > 0) {
             await Promise.all(
-                skippedByTransform.map(async (r) => onSkippedRequest({ url: r.url, reason: 'transform' })),
+                skippedByTransform.map(async (r) => onSkippedRequest(createSkippedRequestArgs(r, 'transform'))),
             );
         }
     }

packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts

@@ -12,6 +12,7 @@ import {
     applyRequestTransform,
     constructUrlPatternObjects,
     createRequestOptions,
+    createSkippedRequestArgs,
     filterRequestOptionsByPatterns,
     urlPatternSchema,
     Request,
@@ -237,6 +238,7 @@ export async function enqueueLinksByClickingElements(
 
     const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
     const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
+    const hasOnSkippedRequest = onSkippedRequest !== undefined;
 
     const urlExcludePatternObjects: UrlPatternObject[] = exclude?.length ? constructUrlPatternObjects(exclude) : [];
     const urlPatternObjects: UrlPatternObject[] = include?.length ? constructUrlPatternObjects(include) : [];
@@ -249,27 +251,31 @@ export async function enqueueLinksByClickingElements(
         clickOptions,
     });
     const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);
-    const skippedByFilters: string[] = [];
+    const skippedByFilters: RequestOptions[] = [];
     let filteredOptions = filterRequestOptionsByPatterns(
         requestOptions,
         urlPatternObjects.length > 0 ? urlPatternObjects : undefined,
         urlExcludePatternObjects,
         undefined,
-        (url) => skippedByFilters.push(url),
+        hasOnSkippedRequest ? (opts) => skippedByFilters.push(opts) : undefined,
     );
 
     if (onSkippedRequest && skippedByFilters.length > 0) {
-        await Promise.all(skippedByFilters.map(async (url) => onSkippedRequest({ url, reason: 'filters' })));
+        await Promise.all(
+            skippedByFilters.map(async (opts) => onSkippedRequest(createSkippedRequestArgs(opts, 'filters'))),
+        );
     }
 
     if (transformRequestFunction) {
         const skippedByTransform: RequestOptions[] = [];
-        filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) =>
-            skippedByTransform.push(r),
+        filteredOptions = applyRequestTransform(
+            filteredOptions,
+            transformRequestFunction,
+            hasOnSkippedRequest ? (r) => skippedByTransform.push(r) : undefined,
         );
         if (onSkippedRequest && skippedByTransform.length > 0) {
             await Promise.all(
-                skippedByTransform.map(async (r) => onSkippedRequest({ url: r.url, reason: 'transform' })),
+                skippedByTransform.map(async (r) => onSkippedRequest(createSkippedRequestArgs(r, 'transform'))),
             );
         }
     }