diff --git a/apps/cli/package.json b/apps/cli/package.json index 0b415f1..c9c04d5 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/nichedb", - "version": "0.56.0", + "version": "0.57.0", "description": "CLI and MCP bridge for NicheDB: browse collections, manage sources and feeds, search items, from any deployment", "type": "module", "main": "src/index.js", diff --git a/apps/cli/src/index.js b/apps/cli/src/index.js index ce636b4..5f56596 100644 --- a/apps/cli/src/index.js +++ b/apps/cli/src/index.js @@ -15,7 +15,7 @@ import { homedir, tmpdir } from 'node:os'; import { join } from 'node:path'; import { createInterface } from 'node:readline'; -export const VERSION = '0.56.0'; +export const VERSION = '0.57.0'; const GEO_FLAGS = ['lat', 'long', 'radius', 'unit', 'bbox', 'sort', 'offset']; const DEFAULT_API = process.env.NICHEDB_API ?? 'https://nichedb.dev'; const CONFIG_DIR = join(process.env.XDG_CONFIG_HOME ?? join(homedir(), '.config'), 'nichedb'); diff --git a/apps/web/package.json b/apps/web/package.json index 6f0b777..93d653f 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/web", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "scripts": { diff --git a/apps/web/src/lib/sites.js b/apps/web/src/lib/sites.js index 0a2bcbb..8429079 100644 --- a/apps/web/src/lib/sites.js +++ b/apps/web/src/lib/sites.js @@ -3,8 +3,11 @@ import { normaliseItem } from '@nichedb/core/adapter'; import { makeHttp } from '@nichedb/core/http'; import { AGENT, + fit, + KINDS, LIST_MAX, parseUrlList, + publicWebUrl, readUrl, recordItem, recordPath, @@ -12,7 +15,7 @@ import { webUrl, } from '@nichedb/core/opensite'; import * as q from '@nichedb/db/queries'; -import { canAddSources, Denied } from './service.js'; +import { canAddSources, Denied, isAdmin } from './service.js'; /** * The operations behind /c/sites, shared by the page, the API and the MCP @@ -26,6 +29,11 @@ import { canAddSources, Denied } from './service.js'; export const COLLECTION = 'sites'; export const PASTED_SOURCE = 'sites-pasted'; +/** Where records a crawler read itself land: POST /api/v1/sites/records. */ +export const CRAWLER_SOURCE = 'pit-crawler'; +/** Records per push. A crawler sends batches; a bigger body is refused, not cut. */ +export const RECORDS_MAX = 200; +const STATUSES = new Set(['live', 'gone', 'moved', 'blocked']); /** How stale a kept record may be before an ask reads the page again. */ export const FRESH_MS = 60 * 60 * 1000; @@ -136,8 +144,15 @@ export async function readAndKeep(url, { fetcher = null } = {}) { const record = await readUrl(asked, { http: fetcher ?? http() }); if (!record) throw new Denied('That address could not be read.', 422); const item = normaliseItem(recordItem(record)); - await q.upsertItems({ collectionId: collection.id, sourceId: source.id, items: [item] }); - const kept = await q.itemByUrls({ collectionId: collection.id, urls: [record.canonical] }); + await q.upsertItems({ + collectionId: collection.id, + sourceId: source.id, + items: [item], + }); + const kept = await q.itemByUrls({ + collectionId: collection.id, + urls: [record.canonical], + }); return { record, item: kept, path: pathOf(record) }; } @@ -153,3 +168,126 @@ export async function recordFor(url, { force = false, fetcher = null } = {}) { } return { ...(await readAndKeep(url, { fetcher })), fresh: false }; } + +const clip = (value, max) => { + const text = typeof value === 'string' ? value.replace(/\s+/g, ' ').trim() : ''; + return text.length > max ? `${text.slice(0, max - 1)}…` : text; +}; +const words = (list, max, len = 40) => + (Array.isArray(list) ? list : []) + .map((t) => clip(String(t ?? ''), len).toLowerCase()) + .filter(Boolean) + .slice(0, max); + +/** + * One pushed record as the row it becomes, or the reason it is refused. + * + * The crawler read the page, so nothing is fetched here; what is checked is + * that the record is one this index would have drawn itself: a public web + * address, a known kind and status, within the spec's size. What the crawler + * learned on top (language, keywords, an excerpt of the text, links counted) + * rides along under `data.enrichment`, and the keywords become `topic:` tags + * and the excerpt the summary when the page has no description, because the + * search column is made of title, summary and tags. + */ +export function crawledItem(entry, { now = new Date() } = {}) { + const record = entry?.record ?? entry; + if (!record || typeof record !== 'object') return { error: 'not a record' }; + const canonical = publicWebUrl(record.canonical ?? record.url); + if (!canonical) return { error: 'canonical is not a public web address' }; + const url = publicWebUrl(record.url) ?? canonical; + const enrichment = + entry?.enrichment && typeof entry.enrichment === 'object' ? entry.enrichment : {}; + const kept = fit({ + ...record, + opensite: String(record.opensite ?? '0.1'), + url, + canonical, + kind: KINDS.includes(record.kind) ? record.kind : 'page', + title: clip(record.title, 300), + description: clip(record.description, 1000) || null, + status: STATUSES.has(record.status) ? record.status : 'live', + source: 'read', + tags: words(record.tags, 20), + fetched_at: Number.isNaN(Date.parse(record.fetched_at)) ? now.toISOString() : record.fetched_at, + }); + const item = recordItem(kept); + if (!item) return { error: 'no canonical address' }; + const keywords = words(enrichment.keywords, 20); + const language = clip(enrichment.language, 12).toLowerCase(); + const excerpt = clip(enrichment.excerpt, 1000); + const extra = { + language: language || null, + keywords, + excerpt: excerpt || null, + words: Number.isFinite(enrichment.words) ? enrichment.words : null, + links: Number.isFinite(enrichment.links) ? enrichment.links : null, + rendered: enrichment.rendered === true, + depth: Number.isFinite(enrichment.depth) ? enrichment.depth : null, + crawler: clip(enrichment.crawler, 80) || null, + }; + const row = normaliseItem({ + ...item, + summary: item.summary || excerpt || null, + tags: [ + ...item.tags, + language ? `lang:${language}` : null, + extra.rendered ? 'rendered' : null, + 'crawled', + ...keywords.map((k) => `topic:${k}`), + ].filter((t, i, all) => t && all.indexOf(t) === i), + data: { ...item.data, enrichment: extra }, + }); + if (!row) return { error: 'no title or address' }; + // The crawler's hash is of the page text, so a re-crawl of an unchanged page writes nothing + // (the upsert skips a row whose hash is the same); fetched_at alone is not a change. + const hash = clip(enrichment.hash, 128); + if (hash) row.contentHash = `crawl:${hash}`; + return { item: row }; +} + +/** + * Keep records a crawler already read: admins only (an admin API key is how a + * crawler signs in), at most RECORDS_MAX a call, each validated on its own so + * one bad record never sinks the batch. Upserted by canonical address into + * the crawler source, so a re-crawl updates the row and an unchanged page + * (same hash) writes nothing. + */ +export async function keepCrawled({ user, records }) { + if (!user) throw new Denied('Sign in with an API key to push records.', 401); + if (!isAdmin(user)) throw new Denied('Pushing crawled records is for the index operators.', 403); + if (!Array.isArray(records) || records.length === 0) + throw new Denied('send { "records": [ { "record": {…}, "enrichment": {…} } ] }', 400); + if (records.length > RECORDS_MAX) + throw new Denied( + `At most ${RECORDS_MAX} records a call; send ${records.length} as batches.`, + 413, + ); + const [collection, source] = await Promise.all([ + q.getCollection(COLLECTION), + q.getSource(CRAWLER_SOURCE), + ]); + if (!collection || !source) + throw new Denied('The crawler source is not set up on this deployment yet.', 503); + const items = []; + const rejected = []; + records.forEach((entry, index) => { + const out = crawledItem(entry); + if (out.item) items.push(out.item); + else rejected.push({ index, error: out.error }); + }); + const { added, updated } = items.length + ? await q.upsertItems({ + collectionId: collection.id, + sourceId: source.id, + items, + }) + : { added: 0, updated: 0 }; + return { + accepted: items.length, + added, + updated, + unchanged: items.length - added - updated, + rejected, + }; +} diff --git a/apps/web/src/routes/sites.js b/apps/web/src/routes/sites.js index c697ea0..7ea6265 100644 --- a/apps/web/src/routes/sites.js +++ b/apps/web/src/routes/sites.js @@ -3,6 +3,7 @@ import { cached, render, respond } from '../lib/http.js'; import { Denied } from '../lib/service.js'; import { findByPath, + keepCrawled, listForHost, pathOf, readAndKeep, @@ -68,7 +69,11 @@ export function registerSites(app) { if (pasted !== '' || file) { const text = `${pasted}\n${file ? await file.text() : ''}`; try { - const out = await submitBulk({ user: c.get('user'), text, name: String(body.name ?? '') }); + const out = await submitBulk({ + user: c.get('user'), + text, + name: String(body.name ?? ''), + }); const notice = `${out.queued} addresses queued${out.rejected.length ? `, ${out.rejected.length} could not be read` : ''}${out.dropped ? `, ${out.dropped} over the cap of ten thousand left out` : ''}. The worker reads them a few hundred a minute; this page shows how far it has got.`; return c.redirect(`${out.page}?notice=${encodeURIComponent(notice)}`, 303); } catch (err) { @@ -150,6 +155,24 @@ export function registerSites(app) { } }); + /** + * Records a crawler already read (pit.world's spider), in batches. Admin + * API key only; see keepCrawled for what is checked. + */ + app.post('/api/v1/sites/records', async (c) => { + const body = await c.req.json().catch(() => null); + try { + const out = await keepCrawled({ + user: c.get('user'), + records: body?.records, + }); + return c.json(out, 200); + } catch (err) { + if (!(err instanceof Denied)) throw err; + return c.json({ error: err.message }, err.status); + } + }); + app.get('/api/v1/sites', async (c) => { const url = (c.req.query('url') ?? '').trim(); if (url === '') return c.json({ error: 'say which address: ?url=' }, 400); @@ -186,7 +209,11 @@ export function registerSites(app) { ); } catch (err) { if (!(err instanceof Denied)) throw err; - return respond(c, { error: err.message, status: err.status, redirectTo: '/c/sites/add' }); + return respond(c, { + error: err.message, + status: err.status, + redirectTo: '/c/sites/add', + }); } }); diff --git a/apps/web/src/routes/sites.test.js b/apps/web/src/routes/sites.test.js index d5a209b..a4ca599 100644 --- a/apps/web/src/routes/sites.test.js +++ b/apps/web/src/routes/sites.test.js @@ -25,7 +25,10 @@ const RECORD = { status: 'live', source: 'read', cards: { - og: { title: 'Inspiring Founders', image: 'https://cdn.example/a.jpg' }, + og: { + title: 'Inspiring Founders', + image: 'https://cdn.example/a.jpg', + }, twitter: { card: 'summary' }, }, jsonld: [], @@ -56,6 +59,7 @@ const store = { return slug === 'sites' ? { id: 1, slug } : null; }, async getSource(slug) { + if (slug === 'pit-crawler') return { id: 3, slug }; return slug === 'sites-pasted' ? { id: 2, slug } : null; }, async itemByUrls({ urls }) { @@ -99,7 +103,14 @@ const { registerSites } = await import('./sites.js'); const { Denied } = await import('../lib/service.js'); const { withModules, decideModules } = await import('../lib/modules.js'); -function app(user = { id: 'u-reader', email: 'r@example.com', role: 'user', timezone: 'UTC' }) { +function app( + user = { + id: 'u-reader', + email: 'r@example.com', + role: 'user', + timezone: 'UTC', + }, +) { const a = new Hono(); a.use('*', async (c, next) => { // A signed-in reader, so pages render directly rather than through the page cache. @@ -191,7 +202,12 @@ describe('the api', () => { }); describe('a whole list', () => { - const admin = { id: 'u-admin', email: 'a@example.com', role: 'admin', timezone: 'UTC' }; + const admin = { + id: 'u-admin', + email: 'a@example.com', + role: 'admin', + timezone: 'UTC', + }; test('a pasted list becomes a source of the submitter, asked to run now, and the page says so', async () => { const before = store.sources.length; @@ -221,9 +237,14 @@ describe('a whole list', () => { const form = new FormData(); form.set( 'file', - new File(['https://c.example/\nhttps://d.example/x\n'], 'list.txt', { type: 'text/plain' }), + new File(['https://c.example/\nhttps://d.example/x\n'], 'list.txt', { + type: 'text/plain', + }), ); - const r = await app(admin).request('/c/sites/add', { method: 'POST', body: form }); + const r = await app(admin).request('/c/sites/add', { + method: 'POST', + body: form, + }); expect(r.status).toBe(303); expect(store.sources.at(-1).config.urls).toEqual(['https://c.example/', 'https://d.example/x']); }); @@ -267,3 +288,75 @@ describe('a whole list', () => { expect((await text.json()).queued).toBe(2); }); }); + +describe('records a crawler pushes', () => { + const admin = { + id: 'u-admin', + email: 'a@example.com', + role: 'admin', + timezone: 'UTC', + }; + const push = (user, body) => + app(user).request('/api/v1/sites/records', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }); + const entry = { + record: { + opensite: '0.1', + url: 'https://crawled.example/a', + canonical: 'https://crawled.example/a', + kind: 'article', + title: 'Wood-fired pizza at home', + status: 'live', + fetched_at: '2026-11-02T00:00:00.000Z', + }, + enrichment: { + language: 'EN', + keywords: ['Pizza', 'oven'], + excerpt: 'How to build a small oven.', + words: 812, + rendered: true, + hash: 'abc123', + crawler: 'pitbot/0.1', + }, + }; + + test('only an admin key may push', async () => { + expect((await push(null, { records: [entry] })).status).toBe(401); + const reader = { + id: 'u-r', + email: 'r@example.com', + role: 'user', + timezone: 'UTC', + }; + expect((await push(reader, { records: [entry] })).status).toBe(403); + }); + + test('a batch is validated record by record and lands in the crawler source', async () => { + store.upserts.length = 0; + const r = await push(admin, { + records: [entry, { record: { canonical: 'http://localhost/x', title: 'x' } }], + }); + expect(r.status).toBe(200); + const body = await r.json(); + expect(body.accepted).toBe(1); + expect(body.rejected).toEqual([{ index: 1, error: 'canonical is not a public web address' }]); + const [row] = store.upserts; + expect(row.url).toBe('https://crawled.example/a'); + expect(row.summary).toBe('How to build a small oven.'); + expect(row.tags).toContain('topic:pizza'); + expect(row.tags).toContain('lang:en'); + expect(row.tags).toContain('site:crawled.example'); + expect(row.tags).toContain('rendered'); + expect(row.contentHash).toBe('crawl:abc123'); + expect(row.data.enrichment.words).toBe(812); + }); + + test('an empty or oversized batch is refused whole', async () => { + expect((await push(admin, { records: [] })).status).toBe(400); + const many = Array.from({ length: 201 }, () => entry); + expect((await push(admin, { records: many })).status).toBe(413); + }); +}); diff --git a/apps/worker/package.json b/apps/worker/package.json index 265a835..69cc0ee 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/worker", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "scripts": { diff --git a/package.json b/package.json index 866f7cd..a3ff022 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "niche-db", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "description": "An open platform for large, ever-growing databases of real-time data: sources in, feeds out. Web, API, CLI and MCP on one Postgres.", diff --git a/packages/adapters/package.json b/packages/adapters/package.json index ac15368..668a7c7 100644 --- a/packages/adapters/package.json +++ b/packages/adapters/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/adapters", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/adapters/src/opensite.js b/packages/adapters/src/opensite.js index 1adccdb..76329db 100644 --- a/packages/adapters/src/opensite.js +++ b/packages/adapters/src/opensite.js @@ -95,11 +95,23 @@ export const opensite = defineAdapter({ config: { urls: [], sitemaps: [], pages: 0 }, enabled: false, }, + { + slug: 'pit-crawler', + name: 'Crawled by pit.world', + description: + 'Pages the pit.world crawler read (up to a thousand per site, JavaScript pages rendered through Obscura) and pushed here with POST /api/v1/sites/records. Nothing to pull: the crawler writes here.', + config: { urls: [], sitemaps: [], pages: 0 }, + enabled: false, + }, ...HOUSE.map(([slug, origin, description]) => ({ slug: `${slug}-pages`, name: `${origin.replace('https://', '')}: its pages`, description, - config: { urls: [`${origin}/`], sitemaps: [`${origin}/sitemap.xml`], pages: DEFAULT_PAGES }, + config: { + urls: [`${origin}/`], + sitemaps: [`${origin}/sitemap.xml`], + pages: DEFAULT_PAGES, + }, enabled: true, })), ], diff --git a/packages/auth/package.json b/packages/auth/package.json index c114617..02b90a0 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/auth", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/config/package.json b/packages/config/package.json index a6fa27f..bba602f 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/config", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/core/package.json b/packages/core/package.json index 1e0a527..67127bf 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/core", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/db/package.json b/packages/db/package.json index cb0ad3e..3711f71 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/db", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/enrichers/package.json b/packages/enrichers/package.json index e675d7a..0e78bf3 100644 --- a/packages/enrichers/package.json +++ b/packages/enrichers/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/enrichers", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/knowledge/package.json b/packages/knowledge/package.json index 1ef9090..c770d08 100644 --- a/packages/knowledge/package.json +++ b/packages/knowledge/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/knowledge", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/notify/package.json b/packages/notify/package.json index 0b1dcc5..493c4c2 100644 --- a/packages/notify/package.json +++ b/packages/notify/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/notify", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/payments/package.json b/packages/payments/package.json index f1befd0..265dd74 100644 --- a/packages/payments/package.json +++ b/packages/payments/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/payments", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/premium/package.json b/packages/premium/package.json index b37b139..d4252f8 100644 --- a/packages/premium/package.json +++ b/packages/premium/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/premium", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/queue/package.json b/packages/queue/package.json index 510da64..78fd647 100644 --- a/packages/queue/package.json +++ b/packages/queue/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/queue", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/test/opensite.test.js b/test/opensite.test.js index 4bde70c..3c1413a 100644 --- a/test/opensite.test.js +++ b/test/opensite.test.js @@ -90,7 +90,11 @@ describe('parsing the head', () => { describe('a record from a page', () => { const url = 'https://nixamp.com/?url=https%3A%2F%2Fserver1.example%3A4321%2Fview%2FK&play=channel%3Aurl-6f4152c1590e'; - const record = readRecord({ url, html: PAGE, fetchedAt: new Date('2026-09-13T04:52:10Z') }); + const record = readRecord({ + url, + html: PAGE, + fetchedAt: new Date('2026-09-13T04:52:10Z'), + }); test('every required key, first answer wins, pictures absolute, tags kept verbatim', () => { expect(record.opensite).toBe('0.1'); @@ -131,7 +135,10 @@ describe('a record from a page', () => { url: 'https://a.example/x', html: '', }); - expect(bad.image).toEqual({ url: 'https://a.example/touch.png', icon: true }); + expect(bad.image).toEqual({ + url: 'https://a.example/touch.png', + icon: true, + }); const none = readRecord({ url: 'https://a.example/x', html: 't', @@ -148,9 +155,16 @@ describe('a record from a page', () => { }); expect(moved.status).toBe('moved'); expect(moved.canonical).toBe('https://new.example/p'); - const gone = readRecord({ url: 'https://a.example/p', status: 410, html: '' }); + const gone = readRecord({ + url: 'https://a.example/p', + status: 410, + html: '', + }); expect(gone.status).toBe('gone'); - const mp3 = readRecord({ url: 'https://a.example/ep.mp3', contentType: 'audio/mpeg' }); + const mp3 = readRecord({ + url: 'https://a.example/ep.mp3', + contentType: 'audio/mpeg', + }); expect(mp3.kind).toBe('audio'); expect(mp3.title).toBe('ep.mp3'); const feed = readRecord({ @@ -167,7 +181,10 @@ describe('a record from a page', () => { url: 'https://a.example/post', html: '', }); - expect(r.author).toEqual({ name: 'Ada', profile: 'https://a.example/~ada/OpenProfile.md' }); + expect(r.author).toEqual({ + name: 'Ada', + profile: 'https://a.example/~ada/OpenProfile.md', + }); const ld = readRecord({ url: 'https://a.example/post', html: '', @@ -259,7 +276,11 @@ describe('refusing politely', () => { body: 'P', type: 'text/html', }, - 'https://closed.example/robots.txt': { body: '', type: 'text/plain', status: 404 }, + 'https://closed.example/robots.txt': { + body: '', + type: 'text/plain', + status: 404, + }, 'https://closed.example/.well-known/opensite.json': { body: JSON.stringify({ opensite: '0.1', @@ -332,9 +353,13 @@ describe('the adapter', () => { expect(COLLECTIONS.some((c) => c.slug === 'sites')).toBe(true); const pasted = a.defaultSources.find((s) => s.slug === 'sites-pasted'); expect(pasted.enabled).toBe(false); + // The crawler pushes into its own source; it has nothing to pull. + expect(a.defaultSources.find((s) => s.slug === 'pit-crawler').enabled).toBe(false); for (const s of a.defaultSources) { for (const k of Object.keys(s.config)) expect(a.configFields.map((f) => f.key)).toContain(k); - expect(s.slug.endsWith('-pages') || s.slug === 'sites-pasted').toBe(true); + expect(s.slug.endsWith('-pages') || ['sites-pasted', 'pit-crawler'].includes(s.slug)).toBe( + true, + ); } const all = ADAPTERS.flatMap((x) => (x.defaultSources ?? []).map((s) => s.slug)); expect(new Set(all).size).toBe(all.length); @@ -431,7 +456,9 @@ describe('a list of addresses', () => { return new Response(''); }, }; - const r = await readUrl('http://169.254.169.254/latest/meta-data', { http }); + const r = await readUrl('http://169.254.169.254/latest/meta-data', { + http, + }); expect(r.status).toBe('blocked'); expect(asked).toEqual([]); });