From d1a254bd01c2fd2409fe33d7544cb9f45bc76f04 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 28 Oct 2026 23:30:21 +0000 Subject: [PATCH] fix(crawl): a one-shot fetch reads the homepage; the directory stays charged on sight x402-gateway 0.13.1 -> 0.16.1. A spoofed/stale/headless string now gets 20 pages an hour per address, but only on the front door (/, /about, /contact, /advertise, /submit, /terms, /privacy): the 2026-09-05 rotation never sent more than two requests from an address, so /topics, /authors and the API keep charging it on sight. Reported 2026-10-28: a one-shot fetch of the homepage got 402. Co-Authored-By: Claude Opus 5.5 --- apps/web/package.json | 2 +- apps/web/src/lib/crawl-gateway.js | 10 ++++++++++ apps/web/test/crawl-gateway.test.js | 11 +++++++++++ bun.lock | 4 ++-- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/apps/web/package.json b/apps/web/package.json index 17ed0cc..fedbb4a 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -20,7 +20,7 @@ "@profullstack/profile": "0.1.6", "@profullstack/rssamplifier": "workspace:*", "@profullstack/submit-feed": "^0.1.3", - "@profullstack/x402-gateway": "^0.13.1", + "@profullstack/x402-gateway": "^0.16.1", "@rssamplifier/auth": "workspace:*", "@rssamplifier/db": "workspace:*", "@rssamplifier/feed": "workspace:*", diff --git a/apps/web/src/lib/crawl-gateway.js b/apps/web/src/lib/crawl-gateway.js index b6c3a9a..a5cac08 100644 --- a/apps/web/src/lib/crawl-gateway.js +++ b/apps/web/src/lib/crawl-gateway.js @@ -189,6 +189,16 @@ export const gateway = createGateway({ * charged like GPTBot: 402 and the offer, a hash instead of a render. */ chargeSpoofedBrowsers: true, + /* + * gateway 0.16 gives a suspect string 20 pages an hour per address before + * the 402. Not here, not everywhere: the rotation above never sent more + * than two requests from one address, so a per-address allowance would hand + * it the whole directory. Only the front door gets it, so a person or an + * agent fetching the homepage once reads it (2026-10-28: a one-shot fetch + * of the homepage got 402) while /topics, /authors and the API stay charged + * on sight. '/' alone is the homepage, not a prefix. + */ + suspectAllowance: { requests: 20, windowSeconds: 3600, paths: ['/', '/about', '/contact', '/advertise', '/submit', '/terms', '/privacy'] }, exempt, /* * What a pass actually buys, printed on the sales page and in every 402. diff --git a/apps/web/test/crawl-gateway.test.js b/apps/web/test/crawl-gateway.test.js index b3acc3c..7d55628 100644 --- a/apps/web/test/crawl-gateway.test.js +++ b/apps/web/test/crawl-gateway.test.js @@ -215,6 +215,17 @@ test('a Chrome string that cannot answer like Chrome is charged', async () => { assert.equal((await gate(request('/authors/somebody', UAS.rotation[2])))?.status, 402); }); +test('a one-shot fetch of the front door reads it; the directory stays charged on sight', async () => { + // 2026-10-28: an agent fetching the homepage once, no cookies, no JS, got + // 402. The front door gets gateway 0.16's per-address allowance; /topics, + // /authors and the API do not, or the rotation above walks back in. + for (const path of ['/', '/about', '/submit']) { + assert.equal(await gate(request(path, UAS.rotation[0], { 'x-forwarded-for': '203.0.113.50' })), undefined, path); + } + assert.equal((await gate(request('/topics/x', UAS.rotation[0], { 'x-forwarded-for': '203.0.113.50' })))?.status, 402); + assert.equal((await gate(request('/', UAS.gptbot)))?.status, 402); +}); + test('the same string with the header a browser sends passes', async () => { for (const ua of UAS.rotation) { assert.equal(await gate(request('/topics/x', ua, { 'sec-fetch-mode': 'navigate' })), undefined, ua); diff --git a/bun.lock b/bun.lock index 4160f1e..ad68a47 100644 --- a/bun.lock +++ b/bun.lock @@ -47,7 +47,7 @@ "@profullstack/profile": "0.1.6", "@profullstack/rssamplifier": "workspace:*", "@profullstack/submit-feed": "^0.1.3", - "@profullstack/x402-gateway": "^0.13.1", + "@profullstack/x402-gateway": "^0.16.1", "@rssamplifier/auth": "workspace:*", "@rssamplifier/db": "workspace:*", "@rssamplifier/feed": "workspace:*", @@ -380,7 +380,7 @@ "@profullstack/x402-client": ["@profullstack/x402-client@0.4.0", "", { "dependencies": { "@noble/curves": "^2.4.0", "@noble/hashes": "^2.4.0", "@profullstack/sealed-secrets": "^0.1.0" }, "bin": { "x402": "bin/x402.js" } }, "sha512-WbPFeey61OH23HZc20/Nr3E2EvA7bhrWMFIqtbzGlsJm0cXOxjM2WIyRDGzi5ssfVQtSwVPvhaUYA66nJOOuhQ=="], - "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.13.1", "", { "dependencies": { "@profullstack/keys": "^0.1.0" } }, "sha512-V7liSO0tShZRlp8dancuDuUSF/rqrr5YF3bzLoZy9wn6ixf1RIRPYTepe72s36JOdSVzi3wfuhaZFm9y0RpV4w=="], + "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.16.1", "", { "dependencies": { "@profullstack/keys": "^0.1.0" } }, "sha512-4aRbH2t5J7vJo9n6SqODNrxD0NBI7GsCo3WjTYuAfIzuPjElHB9TwtpvdpXwOneEJGpQ0JJpHght4M6F39SJng=="], "@radix-ui/number": ["@radix-ui/number@1.1.3", "", {}, "sha512-Road2bidD0uu/1BGDOWNdPI06g0lIRy6IF9GZcIrDK2KGItfor8IQwQa+yM2ERgHM1MmHxaxpTzk0/Jp42lNfA=="],