Skip to content

Commit 44b277c

Browse files
riandradiva18riandradiva18
committed
feat(integrations): add MrScraper tools and block
1 parent f839cc2 commit 44b277c

15 files changed

Lines changed: 1574 additions & 0 deletions

File tree

‎apps/docs/components/icons.tsx‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -650,6 +650,35 @@ export function FirecrawlIcon(props: SVGProps<SVGSVGElement>) {
650650
)
651651
}
652652

653+
interface MrScraperIconProps extends SVGProps<SVGSVGElement> {}
654+
655+
export function MrScraperIcon(props: MrScraperIconProps) {
656+
return (
657+
<svg {...props} viewBox='0 0 180 180' fill='none' xmlns='http://www.w3.org/2000/svg'>
658+
<path
659+
d='M0 0 C1.537 1.145 3.059 2.311 4.562 3.5 C5.306 4.08 6.05 4.66 6.816 5.258 C15.951 12.852 23.403 22.823 28.562 33.5 C28.907 34.191 29.251 34.882 29.605 35.594 C32.633 41.997 34.596 48.71 36.562 55.5 C37.08 57.06 37.08 57.06 37.608 58.652 C38.686 62.999 38.817 66.828 38.758 71.277 C38.755 72.089 38.752 72.9 38.749 73.736 C38.738 76.303 38.713 78.87 38.688 81.438 C38.677 83.188 38.668 84.939 38.66 86.689 C38.64 90.96 38.601 95.23 38.562 99.5 C32.292 99.5 26.023 99.5 19.562 99.5 C19.514 97.413 19.465 95.326 19.414 93.176 C19.36 91.107 19.303 89.038 19.245 86.969 C19.207 85.555 19.172 84.14 19.141 82.725 C18.591 58.512 13.701 37.836 -4.188 20.625 C-20.872 6.604 -40.089 2.252 -61.438 3.5 C-78.655 5.802 -95.262 14.097 -106.438 27.5 C-122.753 49.379 -122.77 73.327 -123.438 99.5 C-129.377 99.5 -135.317 99.5 -141.438 99.5 C-141.53 94.077 -141.61 88.657 -141.657 83.234 C-141.677 81.396 -141.704 79.559 -141.739 77.721 C-142.04 61.535 -140.05 48.063 -132.438 33.5 C-131.946 32.547 -131.455 31.595 -130.949 30.613 C-120.332 11.229 -104.57 -1.841 -84.438 -10.5 C-83.774 -10.803 -83.11 -11.107 -82.426 -11.419 C-54.869 -23.765 -24.073 -16.271 0 0 Z'
660+
fill='#1BD0EF'
661+
transform='translate(141.4375,20.5)'
662+
/>
663+
<path
664+
d='M0 0 C6.93 0 13.86 0 21 0 C21.99 4.455 21.99 4.455 23 9 C25.763 12.491 27.653 14.261 32 15 C34.31 15.103 36.622 15.155 38.934 15.176 C40.32 15.199 41.706 15.223 43.092 15.248 C45.25 15.283 47.408 15.312 49.567 15.326 C64.033 15.435 76.795 17.706 87.895 27.707 C93.752 33.769 99 42.457 99 51 C96.918 51.815 94.834 52.627 92.75 53.438 C91.59 53.89 90.43 54.342 89.234 54.809 C86.81 55.702 84.517 56.458 82 57 C81.381 55.391 81.381 55.391 80.75 53.75 C78.565 48.778 76.466 46.638 71.383 44.426 C64.318 42.148 56.849 42.5 49.5 42.438 C24.82 42.168 24.82 42.168 16 34 C14.197 32.066 12.606 30.108 11 28 C9.948 29.083 9.948 29.083 8.875 30.188 C0.28 38.71 -7.325 42.072 -19.355 42.176 C-20.798 42.199 -22.24 42.223 -23.682 42.248 C-25.928 42.283 -28.174 42.312 -30.421 42.326 C-40.298 42.397 -49.287 42.715 -57.5 48.875 C-59.324 51.46 -60.171 53.961 -61 57 C-64.737 56.195 -68.194 54.824 -71.75 53.438 C-72.92 52.982 -74.091 52.527 -75.297 52.059 C-76.635 51.535 -76.635 51.535 -78 51 C-77.355 41.113 -72.279 33.352 -65.594 26.266 C-53.665 15.846 -39.659 15.672 -24.76 15.544 C-22.641 15.522 -20.522 15.482 -18.402 15.441 C-17.149 15.428 -15.896 15.414 -14.605 15.401 C-9.65 14.85 -5.426 13.711 -2.125 9.75 C-0.624 6.399 -0.624 6.399 0 0 Z'
665+
fill='#1BD0EF'
666+
transform='translate(79,114)'
667+
/>
668+
<path
669+
d='M0 0 C5.167 -0.062 5.167 -0.062 10.438 -0.125 C11.504 -0.143 12.571 -0.161 13.67 -0.18 C28.257 -0.264 39.779 4.354 50.453 14.461 C50.964 14.969 51.474 15.477 52 16 C51.312 19.656 49.746 22.519 47.938 25.75 C47.39 26.735 46.842 27.72 46.277 28.734 C45.645 29.856 45.645 29.856 45 31 C40.498 29.25 36.17 27.208 31.816 25.121 C25.21 22.491 18.143 22.701 11.125 22.438 C10.055 22.394 8.985 22.351 7.883 22.307 C5.255 22.201 2.628 22.1 0 22 C0 14.74 0 7.48 0 0 Z'
670+
fill='#1BD0EF'
671+
transform='translate(98,74)'
672+
/>
673+
<path
674+
d='M0 0 C4.31 2.321 7.197 4.391 9 9 C9.545 13.583 9.587 17.578 7.812 21.875 C4.62 25.618 1.984 28.064 -2.938 28.938 C-7.024 29.261 -9.547 28.276 -13 26 C-16.3 23.146 -17.773 21.161 -18.238 16.73 C-18.407 10.957 -18.106 7.422 -14 3 C-9.367 -0.773 -5.916 -1.721 0 0 Z'
675+
fill='#1BD0EF'
676+
transform='translate(56,72)'
677+
/>
678+
</svg>
679+
)
680+
}
681+
653682
export function JinaAIIcon(props: SVGProps<SVGSVGElement>) {
654683
return (
655684
<svg

‎apps/docs/components/ui/icon-mapping.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,7 @@ import {
192192
ModalIcon,
193193
MondayIcon,
194194
MongoDBIcon,
195+
MrScraperIcon,
195196
MySQLIcon,
196197
Neo4jIcon,
197198
NetSuiteIcon,
@@ -543,6 +544,7 @@ export const blockTypeToIconMap: Record<string, IconComponent> = {
543544
monday: MondayIcon,
544545
mongodb: MongoDBIcon,
545546
mothership: Blimp,
547+
mrscraper: MrScraperIcon,
546548
mssql: MicrosoftSqlIcon,
547549
mysql: MySQLIcon,
548550
neo4j: Neo4jIcon,

‎apps/docs/content/docs/integrations/meta.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,7 @@
178178
"monday",
179179
"monday-service-account",
180180
"mongodb",
181+
"mrscraper",
181182
"mssql",
182183
"mysql",
183184
"neo4j",

‎apps/docs/content/docs/integrations/mrscraper.mdx‎

Lines changed: 252 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,252 @@
1+
---
2+
title: MrScraper
3+
description: Retrieve web pages, search Google, and check account usage
4+
---
5+
6+
import { BlockInfoCard } from "@/components/ui/block-info-card"
7+
8+
<BlockInfoCard
9+
type="mrscraper"
10+
color="#102C35"
11+
/>
12+
13+
{/* MANUAL-CONTENT-START:intro */}
14+
Use [MrScraper](https://mrscraper.com) to retrieve a page as HTML, Markdown, or a screenshot, search Google, and check your account's remaining tokens. Use a MrScraper block for a configured workflow step, or connect the [MrScraper MCP server](#connect-the-mcp-server) to give its tools to an Agent.
15+
16+
## Connect your account
17+
18+
You need a [MrScraper account](https://app.mrscraper.com) and an API token for the native block.
19+
20+
1. Create or copy a token from [MrScraper's API tokens page](https://app.mrscraper.com/api-tokens).
21+
2. In Sim, open **Settings → Secrets** and save the token as `MRSCRAPER_API_KEY` in a namespace available to the workflow's execution actor. See [Secrets](/platform/credentials) for access and runtime resolution.
22+
3. Add a **MrScraper** block to your workflow. Set **API Key** to `{{MRSCRAPER_API_KEY}}`.
23+
4. Select **Get Account** and run the block. Confirm that the account summary shows the expected identity and token balance before running a larger collection.
24+
25+
The API token authenticates requests. Plan tokens measure MrScraper usage; they are a separate balance. Calls use your MrScraper account's quota and limits.
26+
{/* MANUAL-CONTENT-END */}
27+
28+
29+
## Use the block [#usage-instructions]
30+
31+
{/* MANUAL-CONTENT-START:usage */}
32+
### Retrieve a page
33+
34+
Select **Scrape Page** and enter `https://example.com` as the target URL. The default requests HTML. For text to pass into an Agent, enable Markdown and disable HTML if you do not need the markup. Connect the block's `markdown` output to the Agent's prompt or message.
35+
36+
Keep browser rendering set to automatic for the first request. Automatic mode tries without a browser and falls back to browser rendering when needed. Enable browser rendering to force JavaScript execution, or disable it when the non-browser loader returns the content you need. A screenshot forces browser rendering. Real-device routing (`super`) is independent of this choice.
37+
38+
For delayed content, enable browser rendering and supply a CSS selector such as `article` in `waitForSelector`. This waits for an element; it does not click buttons, fill forms, or run a browsing session. Use `homePage` when the site needs its root page visited before the target.
39+
40+
### Find pages with Google search
41+
42+
Select **Google Search** and enter a text query such as `site:example.com documentation`. The default `format` is `json`; use `searchResults` to pass the returned links into later scrape steps. Search results are discovery results, so retrieve the selected pages when you need their full content.
43+
44+
You can also enter a Google search URL:
45+
46+
```text
47+
https://www.google.com/search?q=web+scraping&gl=us&hl=en&start=20
48+
```
49+
50+
This supplies the query `web scraping`, region `us`, language `en`, and page `3`. Explicit `region`, `language`, and `page` inputs override values from the URL. `page` is one based. Select `html` for the Google results page itself, and enable `renderJs` when you need JavaScript-dependent result features.
51+
52+
### Monitor account usage
53+
54+
**Get Account** returns subscription, token usage, remaining quota, rate limits, and account identity. It reads the account associated with the configured API key. It does not report the progress of a scrape or search request.
55+
{/* MANUAL-CONTENT-END */}
56+
57+
58+
Retrieve web pages as HTML, Markdown, or screenshots with browser rendering and geographic routing. Search Google for research and SEO workflows, and check your MrScraper token balance.
59+
60+
## Actions
61+
62+
### mrscraper_scrape
63+
64+
Retrieve a web page as HTML, Markdown, or a screenshot with automatic browser fallback, optional real-device routing, geolocation, and retry controls.
65+
66+
#### Input
67+
68+
| Parameter | Type | Required | Description |
69+
| --------- | ---- | -------- | ----------- |
70+
| `apiKey` | string | Yes | MrScraper API key from https://app.mrscraper.com. |
71+
| `url` | string | Yes | Absolute HTTP or HTTPS URL of the page to retrieve. |
72+
| `html` | boolean | No | Include HTML content. Defaults to true. |
73+
| `markdown` | boolean | No | Include Markdown content. |
74+
| `screenshot` | boolean | No | Capture a JPEG screenshot as a stored file. Always enables browser rendering. |
75+
| `browserRendering` | string | No | auto uses automatic browser fallback; true forces a browser; false disables it. |
76+
| `super` | boolean | No | Use real-device routing independently of browser rendering. |
77+
| `geoCode` | string | No | Proxy country code, such as US, GB, ID, or SG. Defaults to US. |
78+
| `proxyCountry` | string | No | Compatibility country option sent as proxyCountry. |
79+
| `timeout` | number | No | Page timeout in seconds, from 1 to 300. Defaults to 120. |
80+
| `maxRetries` | number | No | Maximum retry count as a nonnegative integer. Zero disables retries. Defaults to 3. |
81+
| `tokenCap` | number | No | Nonnegative retry-token budget. The initial request always runs. |
82+
| `waitForSelector` | string | No | CSS selector to wait for when browser rendering is enabled. |
83+
| `blockResources` | boolean | No | Block images, CSS, and fonts to reduce bandwidth. |
84+
| `homePage` | boolean | No | Visit the site root before loading the target URL. |
85+
| `useProxy` | boolean | No | Allow the service to use a configured proxy. Defaults to true. |
86+
| `proxy` | string | No | Custom proxy in username:password@host:port form. |
87+
| `bypassProxy` | boolean | No | Enable the API proxy-bypass behavior. |
88+
| `cookieJar` | string | No | Cookie configuration, such as domain=example.com&country=us&max_used=100&max_age=50m. |
89+
| `headers` | json | No | Target page request headers as a JSON object with string values. |
90+
91+
#### Output
92+
93+
| Parameter | Type | Description |
94+
| --------- | ---- | ----------- |
95+
| `html` | string | Page HTML, when requested. |
96+
| `markdown` | string | Page Markdown, when requested. |
97+
| `screenshot` | file | JPEG screenshot stored in execution files. |
98+
| `tokenUsage` | number | Plan tokens consumed by the page request. |
99+
| `runtime` | number | Page request runtime in seconds. |
100+
| `retryCount` | number | Number of retries performed. |
101+
| `targetStatusCode` | number | Target page HTTP status from x-status-code. |
102+
| `error` | string | Page retrieval error, when reported by MrScraper. |
103+
104+
### mrscraper_search
105+
106+
Search Google from query text or a Google search URL and return organic results, search metadata, pagination, or the raw results-page HTML.
107+
108+
#### Input
109+
110+
| Parameter | Type | Required | Description |
111+
| --------- | ---- | -------- | ----------- |
112+
| `apiKey` | string | Yes | MrScraper API key from https://app.mrscraper.com. |
113+
| `query` | string | Yes | Search query or Google search URL with q, gl, hl, and start parameters. |
114+
| `format` | string | No | json returns parsed results; html returns results-page HTML. Defaults to json. |
115+
| `region` | string | No | Country code such as us, gb, or id. Overrides gl from a Google URL. |
116+
| `language` | string | No | Language code such as en, es, or id. Overrides hl from a Google URL. |
117+
| `page` | number | No | One-based result page. Overrides the page derived from a Google URL start parameter. |
118+
| `renderJs` | boolean | No | Wait for JavaScript-dependent search features such as AI Overview to load. |
119+
120+
#### Output
121+
122+
| Parameter | Type | Description |
123+
| --------- | ---- | ----------- |
124+
| `searchResults` | array | Organic Google results in ranking order. Empty for HTML format. |
125+
| ↳ `title` | string | Result title. |
126+
| ↳ `link` | string | Destination URL. |
127+
| ↳ `description` | string | Result description or snippet. |
128+
| ↳ `source` | string | Source name. |
129+
| ↳ `displayLink` | string | Displayed URL or breadcrumb. |
130+
| ↳ `rank` | number | Position among organic results. |
131+
| ↳ `globalRank` | number | Position among all search result features. |
132+
| `html` | string | Raw results-page HTML when format is html. |
133+
| `general` | object | Search metadata for parsed JSON results. |
134+
| ↳ `query` | string | Search query. |
135+
| ↳ `resultsCount` | number | Number of results reported by the parser. |
136+
| ↳ `searchTime` | number | Search time reported by Google in seconds. |
137+
| ↳ `language` | string | Search language. |
138+
| ↳ `location` | string | Search country. |
139+
| ↳ `pageTitle` | string | Results-page title. |
140+
| ↳ `timestamp` | string | Search response timestamp. |
141+
| `pagination` | object | Current page and links for continuing a parsed search. |
142+
| ↳ `currentPage` | number | Current result page. |
143+
| ↳ `nextPage` | number | Next result page. |
144+
| ↳ `nextPageStart` | number | Google start offset of the next page. |
145+
| ↳ `nextPageLink` | string | Google URL for the next page. |
146+
| ↳ `pages` | array | Available result pages. |
147+
| ↳ `page` | number | One-based page number. |
148+
| ↳ `start` | number | Google start offset. |
149+
| ↳ `link` | string | Google result-page URL. |
150+
| `tokenUsage` | number | Plan tokens consumed by the search request. |
151+
152+
### mrscraper_get_account
153+
154+
Check account identity, subscription status, token usage, remaining quota, and rate limits.
155+
156+
#### Input
157+
158+
| Parameter | Type | Required | Description |
159+
| --------- | ---- | -------- | ----------- |
160+
| `apiKey` | string | Yes | MrScraper API key from https://app.mrscraper.com. |
161+
162+
#### Output
163+
164+
| Parameter | Type | Description |
165+
| --------- | ---- | ----------- |
166+
| `subscriptionStatus` | string | Current subscription status. |
167+
| `enterprise` | boolean | Whether the account has an enterprise plan. |
168+
| `tokenUsage` | number | Plan tokens already consumed. |
169+
| `tokenLimit` | number | Plan token allowance. |
170+
| `tokenRemaining` | number | Remaining plan tokens, clamped to zero. |
171+
| `usagePercent` | number | Percentage of the token allowance consumed. |
172+
| `rateLimit` | number | Account request rate limit. |
173+
| `rateTtl` | number | Rate-limit window reported by the service. |
174+
| `autoRenew` | boolean | Whether the subscription renews automatically. |
175+
| `endsAt` | string | Subscription end timestamp, when present. |
176+
| `user` | object | Account identity. |
177+
| ↳ `name` | string | Account holder name. |
178+
| ↳ `email` | string | Account holder email. |
179+
| ↳ `verified` | boolean | Whether the account holder is verified. |
180+
181+
{/* MANUAL-CONTENT-START:notes */}
182+
## Work with the results
183+
184+
The action tables above describe Sim's outputs. For scraping, `html` and `markdown` contain the requested page content. When requested, `screenshot` is a stored file that you can pass to a later file-compatible block. Request it only when you need an image; screenshots can be large.
185+
186+
Check `targetStatusCode`, `error`, and the returned content before processing a scrape. The target page can return an error or a challenge page even when MrScraper accepted the request. `tokenUsage`, `runtime`, and `retryCount` help explain how the request completed. Format and site behavior determine which content fields are populated.
187+
188+
For JSON search results, use `searchResults` for organic results, `general` for query metadata, and `pagination` for available result pages. An empty results array is a valid outcome. For HTML search, use the `html` output. These are search results, not a guarantee that each linked page is accessible.
189+
190+
The account action returns a selected summary. API tokens and billing identifiers from the provider's account record are excluded.
191+
192+
## Control time and token usage
193+
194+
Scrape `timeout` is a page-loading timeout in seconds, from `1` to `300`, with a default of `120`. It is separate from workflow execution limits. `maxRetries` defaults to `3`; set it to `0` to disable retries.
195+
196+
`tokenCap` stops further retries when the running token total reaches the cap. The first attempt still runs even if it exceeds the cap, so this setting does not guarantee a maximum charge. Web fetching uses runtime and bandwidth: one plan token per 30 seconds and one per 0.25 MB, rounded up for each component. See MrScraper's [Token Plan](https://docs.mrscraper.com/docs/getting-started/api-token) for the maintained calculation.
197+
198+
The combined API response must fit Sim's 10 MiB tool response limit, including the screenshot's base64 representation before it is stored as a file. If the response is too large, request fewer formats or omit the screenshot.
199+
200+
Enable `blockResources` for text collection when blocking images, CSS, and fonts will not remove content you need. Use `geoCode` for country-specific page routing. `proxy`, `cookieJar`, and target `headers` are advanced request controls; supply secret references for sensitive values. Target headers are sent to the page, separately from the API key used to authenticate with MrScraper.
201+
202+
## Handle failed requests
203+
204+
| Result | What to check |
205+
| --- | --- |
206+
| Authentication fails | Confirm the API token is current and the execution actor can resolve its Sim secret. |
207+
| A page is incomplete | Inspect the content, then change browser rendering or add a selector wait for the missing element. |
208+
| A page is blocked or varies by location | Inspect the target status and content, then try the needed geographic or real-device routing setting. |
209+
| A request times out | Check the page-loading timeout and selector. A selector that never appears can exhaust the timeout. |
210+
| Quota or rate limits are reached | Run the account action and review the token balance and limits in MrScraper. |
211+
| Search results are empty or irrelevant | Refine the query, region, language, or page. |
212+
213+
Change one request setting at a time after inspecting a failed result. Repeating a scrape can consume additional tokens. The block retrieves page content and Google results; it does not manage saved AI scrapers or stored scraper runs in the MrScraper dashboard.
214+
215+
## Connect the MCP server
216+
217+
You can also connect the hosted [MrScraper MCP server](https://docs.mrscraper.com/docs/getting-started/mcp-server) through Sim's external MCP support.
218+
219+
1. With workspace **Write** or **Admin** access, open **Settings → MCP tools** and select **Add server**.
220+
2. Enter `MrScraper` as **Server Name** and `https://mcp.mrscraper.com/mcp` as **Server URL**. Leave **Headers** empty for OAuth.
221+
3. Select **Add server** and complete MrScraper's OAuth browser sign-in. If needed, select **Authorize** on the saved server.
222+
4. Inspect the discovered tools, then add the operations you need under an Agent's **Tools**, or select the server in an **MCP** block.
223+
224+
**Edit JSON** accepts this configuration:
225+
226+
```json
227+
{
228+
"mcpServers": {
229+
"mrscraper": {
230+
"url": "https://mcp.mrscraper.com/mcp"
231+
}
232+
}
233+
}
234+
```
235+
236+
| MCP tool | Purpose | OAuth scope |
237+
| --- | --- | --- |
238+
| `scraperAPI` | Retrieve a page | `scrape:read` |
239+
| `webSearch` | Search Google | `scrape:read` |
240+
| `status` | Read account usage | `account:read` |
241+
242+
Call `status` with `{}` to verify account access. Reauthorize for `403 insufficient_scope`. MCP operations use their discovered schemas and response envelopes; their argument names and outputs differ from the native block. See [Using MCP tools](/agents/mcp) for access policies, connection refresh, permissions, and self-hosted network configuration.
243+
244+
## Related docs
245+
246+
- [Secrets](/platform/credentials)
247+
- [Using MCP tools](/agents/mcp)
248+
- [Agent block](/workflows/blocks/agent)
249+
- [MrScraper API reference](https://docs.mrscraper.com/docs/api/overview)
250+
- [MrScraper MCP server](https://docs.mrscraper.com/docs/getting-started/mcp-server)
251+
- [MrScraper Token Plan](https://docs.mrscraper.com/docs/getting-started/api-token)
252+
{/* MANUAL-CONTENT-END */}

0 commit comments

Comments
 (0)