From 580bf42ca31c59d2c8e4b15f91d830985f1185e4 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 2 Nov 2026 04:13:27 +0000 Subject: [PATCH] Sites: POST /api/v1/sites/records for records a crawler already read (0.57.0) pit.world's crawler reads pages itself (up to 1,000 per site, JavaScript pages through Obscura) and pushes OpenSite records here in batches of up to 200. Admin API key only. Each record is validated on its own (public address, known kind/status, spec size) and upserted by canonical address into a new disabled source, pit-crawler. Enrichment rides along under data.enrichment; keywords become topic: tags and the excerpt the summary, since search is title + summary + tags. The crawler's text hash becomes the content hash, so a re-crawl of an unchanged page writes nothing. Co-Authored-By: Claude Opus 5.5 --- apps/cli/package.json | 2 +- apps/cli/src/index.js | 2 +- apps/web/package.json | 2 +- apps/web/src/lib/sites.js | 144 +++++++++++++++++++++++++++++- apps/web/src/routes/sites.js | 31 ++++++- apps/web/src/routes/sites.test.js | 103 +++++++++++++++++++-- apps/worker/package.json | 2 +- package.json | 2 +- packages/adapters/package.json | 2 +- packages/adapters/src/opensite.js | 14 ++- packages/auth/package.json | 2 +- packages/config/package.json | 2 +- packages/core/package.json | 2 +- packages/db/package.json | 2 +- packages/enrichers/package.json | 2 +- packages/knowledge/package.json | 2 +- packages/notify/package.json | 2 +- packages/payments/package.json | 2 +- packages/premium/package.json | 2 +- packages/queue/package.json | 2 +- test/opensite.test.js | 43 +++++++-- 21 files changed, 332 insertions(+), 35 deletions(-) diff --git a/apps/cli/package.json b/apps/cli/package.json index 0b415f1f..c9c04d52 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/nichedb", - "version": "0.56.0", + "version": "0.57.0", "description": "CLI and MCP bridge for NicheDB: browse collections, manage sources and feeds, search items, from any deployment", "type": "module", "main": "src/index.js", diff --git a/apps/cli/src/index.js b/apps/cli/src/index.js index ce636b4d..5f565965 100644 --- a/apps/cli/src/index.js +++ b/apps/cli/src/index.js @@ -15,7 +15,7 @@ import { homedir, tmpdir } from 'node:os'; import { join } from 'node:path'; import { createInterface } from 'node:readline'; -export const VERSION = '0.56.0'; +export const VERSION = '0.57.0'; const GEO_FLAGS = ['lat', 'long', 'radius', 'unit', 'bbox', 'sort', 'offset']; const DEFAULT_API = process.env.NICHEDB_API ?? 'https://nichedb.dev'; const CONFIG_DIR = join(process.env.XDG_CONFIG_HOME ?? join(homedir(), '.config'), 'nichedb'); diff --git a/apps/web/package.json b/apps/web/package.json index 6f0b7775..93d653f7 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/web", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "scripts": { diff --git a/apps/web/src/lib/sites.js b/apps/web/src/lib/sites.js index 0a2bcbb8..84290792 100644 --- a/apps/web/src/lib/sites.js +++ b/apps/web/src/lib/sites.js @@ -3,8 +3,11 @@ import { normaliseItem } from '@nichedb/core/adapter'; import { makeHttp } from '@nichedb/core/http'; import { AGENT, + fit, + KINDS, LIST_MAX, parseUrlList, + publicWebUrl, readUrl, recordItem, recordPath, @@ -12,7 +15,7 @@ import { webUrl, } from '@nichedb/core/opensite'; import * as q from '@nichedb/db/queries'; -import { canAddSources, Denied } from './service.js'; +import { canAddSources, Denied, isAdmin } from './service.js'; /** * The operations behind /c/sites, shared by the page, the API and the MCP @@ -26,6 +29,11 @@ import { canAddSources, Denied } from './service.js'; export const COLLECTION = 'sites'; export const PASTED_SOURCE = 'sites-pasted'; +/** Where records a crawler read itself land: POST /api/v1/sites/records. */ +export const CRAWLER_SOURCE = 'pit-crawler'; +/** Records per push. A crawler sends batches; a bigger body is refused, not cut. */ +export const RECORDS_MAX = 200; +const STATUSES = new Set(['live', 'gone', 'moved', 'blocked']); /** How stale a kept record may be before an ask reads the page again. */ export const FRESH_MS = 60 * 60 * 1000; @@ -136,8 +144,15 @@ export async function readAndKeep(url, { fetcher = null } = {}) { const record = await readUrl(asked, { http: fetcher ?? http() }); if (!record) throw new Denied('That address could not be read.', 422); const item = normaliseItem(recordItem(record)); - await q.upsertItems({ collectionId: collection.id, sourceId: source.id, items: [item] }); - const kept = await q.itemByUrls({ collectionId: collection.id, urls: [record.canonical] }); + await q.upsertItems({ + collectionId: collection.id, + sourceId: source.id, + items: [item], + }); + const kept = await q.itemByUrls({ + collectionId: collection.id, + urls: [record.canonical], + }); return { record, item: kept, path: pathOf(record) }; } @@ -153,3 +168,126 @@ export async function recordFor(url, { force = false, fetcher = null } = {}) { } return { ...(await readAndKeep(url, { fetcher })), fresh: false }; } + +const clip = (value, max) => { + const text = typeof value === 'string' ? value.replace(/\s+/g, ' ').trim() : ''; + return text.length > max ? `${text.slice(0, max - 1)}…` : text; +}; +const words = (list, max, len = 40) => + (Array.isArray(list) ? list : []) + .map((t) => clip(String(t ?? ''), len).toLowerCase()) + .filter(Boolean) + .slice(0, max); + +/** + * One pushed record as the row it becomes, or the reason it is refused. + * + * The crawler read the page, so nothing is fetched here; what is checked is + * that the record is one this index would have drawn itself: a public web + * address, a known kind and status, within the spec's size. What the crawler + * learned on top (language, keywords, an excerpt of the text, links counted) + * rides along under `data.enrichment`, and the keywords become `topic:` tags + * and the excerpt the summary when the page has no description, because the + * search column is made of title, summary and tags. + */ +export function crawledItem(entry, { now = new Date() } = {}) { + const record = entry?.record ?? entry; + if (!record || typeof record !== 'object') return { error: 'not a record' }; + const canonical = publicWebUrl(record.canonical ?? record.url); + if (!canonical) return { error: 'canonical is not a public web address' }; + const url = publicWebUrl(record.url) ?? canonical; + const enrichment = + entry?.enrichment && typeof entry.enrichment === 'object' ? entry.enrichment : {}; + const kept = fit({ + ...record, + opensite: String(record.opensite ?? '0.1'), + url, + canonical, + kind: KINDS.includes(record.kind) ? record.kind : 'page', + title: clip(record.title, 300), + description: clip(record.description, 1000) || null, + status: STATUSES.has(record.status) ? record.status : 'live', + source: 'read', + tags: words(record.tags, 20), + fetched_at: Number.isNaN(Date.parse(record.fetched_at)) ? now.toISOString() : record.fetched_at, + }); + const item = recordItem(kept); + if (!item) return { error: 'no canonical address' }; + const keywords = words(enrichment.keywords, 20); + const language = clip(enrichment.language, 12).toLowerCase(); + const excerpt = clip(enrichment.excerpt, 1000); + const extra = { + language: language || null, + keywords, + excerpt: excerpt || null, + words: Number.isFinite(enrichment.words) ? enrichment.words : null, + links: Number.isFinite(enrichment.links) ? enrichment.links : null, + rendered: enrichment.rendered === true, + depth: Number.isFinite(enrichment.depth) ? enrichment.depth : null, + crawler: clip(enrichment.crawler, 80) || null, + }; + const row = normaliseItem({ + ...item, + summary: item.summary || excerpt || null, + tags: [ + ...item.tags, + language ? `lang:${language}` : null, + extra.rendered ? 'rendered' : null, + 'crawled', + ...keywords.map((k) => `topic:${k}`), + ].filter((t, i, all) => t && all.indexOf(t) === i), + data: { ...item.data, enrichment: extra }, + }); + if (!row) return { error: 'no title or address' }; + // The crawler's hash is of the page text, so a re-crawl of an unchanged page writes nothing + // (the upsert skips a row whose hash is the same); fetched_at alone is not a change. + const hash = clip(enrichment.hash, 128); + if (hash) row.contentHash = `crawl:${hash}`; + return { item: row }; +} + +/** + * Keep records a crawler already read: admins only (an admin API key is how a + * crawler signs in), at most RECORDS_MAX a call, each validated on its own so + * one bad record never sinks the batch. Upserted by canonical address into + * the crawler source, so a re-crawl updates the row and an unchanged page + * (same hash) writes nothing. + */ +export async function keepCrawled({ user, records }) { + if (!user) throw new Denied('Sign in with an API key to push records.', 401); + if (!isAdmin(user)) throw new Denied('Pushing crawled records is for the index operators.', 403); + if (!Array.isArray(records) || records.length === 0) + throw new Denied('send { "records": [ { "record": {…}, "enrichment": {…} } ] }', 400); + if (records.length > RECORDS_MAX) + throw new Denied( + `At most ${RECORDS_MAX} records a call; send ${records.length} as batches.`, + 413, + ); + const [collection, source] = await Promise.all([ + q.getCollection(COLLECTION), + q.getSource(CRAWLER_SOURCE), + ]); + if (!collection || !source) + throw new Denied('The crawler source is not set up on this deployment yet.', 503); + const items = []; + const rejected = []; + records.forEach((entry, index) => { + const out = crawledItem(entry); + if (out.item) items.push(out.item); + else rejected.push({ index, error: out.error }); + }); + const { added, updated } = items.length + ? await q.upsertItems({ + collectionId: collection.id, + sourceId: source.id, + items, + }) + : { added: 0, updated: 0 }; + return { + accepted: items.length, + added, + updated, + unchanged: items.length - added - updated, + rejected, + }; +} diff --git a/apps/web/src/routes/sites.js b/apps/web/src/routes/sites.js index c697ea0b..7ea62650 100644 --- a/apps/web/src/routes/sites.js +++ b/apps/web/src/routes/sites.js @@ -3,6 +3,7 @@ import { cached, render, respond } from '../lib/http.js'; import { Denied } from '../lib/service.js'; import { findByPath, + keepCrawled, listForHost, pathOf, readAndKeep, @@ -68,7 +69,11 @@ export function registerSites(app) { if (pasted !== '' || file) { const text = `${pasted}\n${file ? await file.text() : ''}`; try { - const out = await submitBulk({ user: c.get('user'), text, name: String(body.name ?? '') }); + const out = await submitBulk({ + user: c.get('user'), + text, + name: String(body.name ?? ''), + }); const notice = `${out.queued} addresses queued${out.rejected.length ? `, ${out.rejected.length} could not be read` : ''}${out.dropped ? `, ${out.dropped} over the cap of ten thousand left out` : ''}. The worker reads them a few hundred a minute; this page shows how far it has got.`; return c.redirect(`${out.page}?notice=${encodeURIComponent(notice)}`, 303); } catch (err) { @@ -150,6 +155,24 @@ export function registerSites(app) { } }); + /** + * Records a crawler already read (pit.world's spider), in batches. Admin + * API key only; see keepCrawled for what is checked. + */ + app.post('/api/v1/sites/records', async (c) => { + const body = await c.req.json().catch(() => null); + try { + const out = await keepCrawled({ + user: c.get('user'), + records: body?.records, + }); + return c.json(out, 200); + } catch (err) { + if (!(err instanceof Denied)) throw err; + return c.json({ error: err.message }, err.status); + } + }); + app.get('/api/v1/sites', async (c) => { const url = (c.req.query('url') ?? '').trim(); if (url === '') return c.json({ error: 'say which address: ?url=' }, 400); @@ -186,7 +209,11 @@ export function registerSites(app) { ); } catch (err) { if (!(err instanceof Denied)) throw err; - return respond(c, { error: err.message, status: err.status, redirectTo: '/c/sites/add' }); + return respond(c, { + error: err.message, + status: err.status, + redirectTo: '/c/sites/add', + }); } }); diff --git a/apps/web/src/routes/sites.test.js b/apps/web/src/routes/sites.test.js index d5a209b6..a4ca5992 100644 --- a/apps/web/src/routes/sites.test.js +++ b/apps/web/src/routes/sites.test.js @@ -25,7 +25,10 @@ const RECORD = { status: 'live', source: 'read', cards: { - og: { title: 'Inspiring Founders', image: 'https://cdn.example/a.jpg' }, + og: { + title: 'Inspiring Founders', + image: 'https://cdn.example/a.jpg', + }, twitter: { card: 'summary' }, }, jsonld: [], @@ -56,6 +59,7 @@ const store = { return slug === 'sites' ? { id: 1, slug } : null; }, async getSource(slug) { + if (slug === 'pit-crawler') return { id: 3, slug }; return slug === 'sites-pasted' ? { id: 2, slug } : null; }, async itemByUrls({ urls }) { @@ -99,7 +103,14 @@ const { registerSites } = await import('./sites.js'); const { Denied } = await import('../lib/service.js'); const { withModules, decideModules } = await import('../lib/modules.js'); -function app(user = { id: 'u-reader', email: 'r@example.com', role: 'user', timezone: 'UTC' }) { +function app( + user = { + id: 'u-reader', + email: 'r@example.com', + role: 'user', + timezone: 'UTC', + }, +) { const a = new Hono(); a.use('*', async (c, next) => { // A signed-in reader, so pages render directly rather than through the page cache. @@ -191,7 +202,12 @@ describe('the api', () => { }); describe('a whole list', () => { - const admin = { id: 'u-admin', email: 'a@example.com', role: 'admin', timezone: 'UTC' }; + const admin = { + id: 'u-admin', + email: 'a@example.com', + role: 'admin', + timezone: 'UTC', + }; test('a pasted list becomes a source of the submitter, asked to run now, and the page says so', async () => { const before = store.sources.length; @@ -221,9 +237,14 @@ describe('a whole list', () => { const form = new FormData(); form.set( 'file', - new File(['https://c.example/\nhttps://d.example/x\n'], 'list.txt', { type: 'text/plain' }), + new File(['https://c.example/\nhttps://d.example/x\n'], 'list.txt', { + type: 'text/plain', + }), ); - const r = await app(admin).request('/c/sites/add', { method: 'POST', body: form }); + const r = await app(admin).request('/c/sites/add', { + method: 'POST', + body: form, + }); expect(r.status).toBe(303); expect(store.sources.at(-1).config.urls).toEqual(['https://c.example/', 'https://d.example/x']); }); @@ -267,3 +288,75 @@ describe('a whole list', () => { expect((await text.json()).queued).toBe(2); }); }); + +describe('records a crawler pushes', () => { + const admin = { + id: 'u-admin', + email: 'a@example.com', + role: 'admin', + timezone: 'UTC', + }; + const push = (user, body) => + app(user).request('/api/v1/sites/records', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }); + const entry = { + record: { + opensite: '0.1', + url: 'https://crawled.example/a', + canonical: 'https://crawled.example/a', + kind: 'article', + title: 'Wood-fired pizza at home', + status: 'live', + fetched_at: '2026-11-02T00:00:00.000Z', + }, + enrichment: { + language: 'EN', + keywords: ['Pizza', 'oven'], + excerpt: 'How to build a small oven.', + words: 812, + rendered: true, + hash: 'abc123', + crawler: 'pitbot/0.1', + }, + }; + + test('only an admin key may push', async () => { + expect((await push(null, { records: [entry] })).status).toBe(401); + const reader = { + id: 'u-r', + email: 'r@example.com', + role: 'user', + timezone: 'UTC', + }; + expect((await push(reader, { records: [entry] })).status).toBe(403); + }); + + test('a batch is validated record by record and lands in the crawler source', async () => { + store.upserts.length = 0; + const r = await push(admin, { + records: [entry, { record: { canonical: 'http://localhost/x', title: 'x' } }], + }); + expect(r.status).toBe(200); + const body = await r.json(); + expect(body.accepted).toBe(1); + expect(body.rejected).toEqual([{ index: 1, error: 'canonical is not a public web address' }]); + const [row] = store.upserts; + expect(row.url).toBe('https://crawled.example/a'); + expect(row.summary).toBe('How to build a small oven.'); + expect(row.tags).toContain('topic:pizza'); + expect(row.tags).toContain('lang:en'); + expect(row.tags).toContain('site:crawled.example'); + expect(row.tags).toContain('rendered'); + expect(row.contentHash).toBe('crawl:abc123'); + expect(row.data.enrichment.words).toBe(812); + }); + + test('an empty or oversized batch is refused whole', async () => { + expect((await push(admin, { records: [] })).status).toBe(400); + const many = Array.from({ length: 201 }, () => entry); + expect((await push(admin, { records: many })).status).toBe(413); + }); +}); diff --git a/apps/worker/package.json b/apps/worker/package.json index 265a835b..69cc0eef 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/worker", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "scripts": { diff --git a/package.json b/package.json index 866f7cd3..a3ff0222 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "niche-db", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "description": "An open platform for large, ever-growing databases of real-time data: sources in, feeds out. Web, API, CLI and MCP on one Postgres.", diff --git a/packages/adapters/package.json b/packages/adapters/package.json index ac15368c..668a7c7a 100644 --- a/packages/adapters/package.json +++ b/packages/adapters/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/adapters", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/adapters/src/opensite.js b/packages/adapters/src/opensite.js index 1adccdbb..76329dba 100644 --- a/packages/adapters/src/opensite.js +++ b/packages/adapters/src/opensite.js @@ -95,11 +95,23 @@ export const opensite = defineAdapter({ config: { urls: [], sitemaps: [], pages: 0 }, enabled: false, }, + { + slug: 'pit-crawler', + name: 'Crawled by pit.world', + description: + 'Pages the pit.world crawler read (up to a thousand per site, JavaScript pages rendered through Obscura) and pushed here with POST /api/v1/sites/records. Nothing to pull: the crawler writes here.', + config: { urls: [], sitemaps: [], pages: 0 }, + enabled: false, + }, ...HOUSE.map(([slug, origin, description]) => ({ slug: `${slug}-pages`, name: `${origin.replace('https://', '')}: its pages`, description, - config: { urls: [`${origin}/`], sitemaps: [`${origin}/sitemap.xml`], pages: DEFAULT_PAGES }, + config: { + urls: [`${origin}/`], + sitemaps: [`${origin}/sitemap.xml`], + pages: DEFAULT_PAGES, + }, enabled: true, })), ], diff --git a/packages/auth/package.json b/packages/auth/package.json index c1146175..02b90a0d 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/auth", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/config/package.json b/packages/config/package.json index a6fa27fa..bba602f7 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/config", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/core/package.json b/packages/core/package.json index 1e0a5274..67127bfb 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/core", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/db/package.json b/packages/db/package.json index cb0ad3e2..3711f719 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/db", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/enrichers/package.json b/packages/enrichers/package.json index e675d7a7..0e78bf32 100644 --- a/packages/enrichers/package.json +++ b/packages/enrichers/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/enrichers", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/knowledge/package.json b/packages/knowledge/package.json index 1ef90902..c770d08d 100644 --- a/packages/knowledge/package.json +++ b/packages/knowledge/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/knowledge", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/notify/package.json b/packages/notify/package.json index 0b1dcc52..493c4c21 100644 --- a/packages/notify/package.json +++ b/packages/notify/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/notify", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/payments/package.json b/packages/payments/package.json index f1befd08..265dd747 100644 --- a/packages/payments/package.json +++ b/packages/payments/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/payments", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/premium/package.json b/packages/premium/package.json index b37b139c..d4252f8a 100644 --- a/packages/premium/package.json +++ b/packages/premium/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/premium", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/packages/queue/package.json b/packages/queue/package.json index 510da641..78fd647a 100644 --- a/packages/queue/package.json +++ b/packages/queue/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/queue", - "version": "0.56.0", + "version": "0.57.0", "private": true, "type": "module", "exports": { diff --git a/test/opensite.test.js b/test/opensite.test.js index 4bde70c2..3c1413a0 100644 --- a/test/opensite.test.js +++ b/test/opensite.test.js @@ -90,7 +90,11 @@ describe('parsing the head', () => { describe('a record from a page', () => { const url = 'https://nixamp.com/?url=https%3A%2F%2Fserver1.example%3A4321%2Fview%2FK&play=channel%3Aurl-6f4152c1590e'; - const record = readRecord({ url, html: PAGE, fetchedAt: new Date('2026-09-13T04:52:10Z') }); + const record = readRecord({ + url, + html: PAGE, + fetchedAt: new Date('2026-09-13T04:52:10Z'), + }); test('every required key, first answer wins, pictures absolute, tags kept verbatim', () => { expect(record.opensite).toBe('0.1'); @@ -131,7 +135,10 @@ describe('a record from a page', () => { url: 'https://a.example/x', html: '', }); - expect(bad.image).toEqual({ url: 'https://a.example/touch.png', icon: true }); + expect(bad.image).toEqual({ + url: 'https://a.example/touch.png', + icon: true, + }); const none = readRecord({ url: 'https://a.example/x', html: 't', @@ -148,9 +155,16 @@ describe('a record from a page', () => { }); expect(moved.status).toBe('moved'); expect(moved.canonical).toBe('https://new.example/p'); - const gone = readRecord({ url: 'https://a.example/p', status: 410, html: '' }); + const gone = readRecord({ + url: 'https://a.example/p', + status: 410, + html: '', + }); expect(gone.status).toBe('gone'); - const mp3 = readRecord({ url: 'https://a.example/ep.mp3', contentType: 'audio/mpeg' }); + const mp3 = readRecord({ + url: 'https://a.example/ep.mp3', + contentType: 'audio/mpeg', + }); expect(mp3.kind).toBe('audio'); expect(mp3.title).toBe('ep.mp3'); const feed = readRecord({ @@ -167,7 +181,10 @@ describe('a record from a page', () => { url: 'https://a.example/post', html: '', }); - expect(r.author).toEqual({ name: 'Ada', profile: 'https://a.example/~ada/OpenProfile.md' }); + expect(r.author).toEqual({ + name: 'Ada', + profile: 'https://a.example/~ada/OpenProfile.md', + }); const ld = readRecord({ url: 'https://a.example/post', html: '', @@ -259,7 +276,11 @@ describe('refusing politely', () => { body: 'P', type: 'text/html', }, - 'https://closed.example/robots.txt': { body: '', type: 'text/plain', status: 404 }, + 'https://closed.example/robots.txt': { + body: '', + type: 'text/plain', + status: 404, + }, 'https://closed.example/.well-known/opensite.json': { body: JSON.stringify({ opensite: '0.1', @@ -332,9 +353,13 @@ describe('the adapter', () => { expect(COLLECTIONS.some((c) => c.slug === 'sites')).toBe(true); const pasted = a.defaultSources.find((s) => s.slug === 'sites-pasted'); expect(pasted.enabled).toBe(false); + // The crawler pushes into its own source; it has nothing to pull. + expect(a.defaultSources.find((s) => s.slug === 'pit-crawler').enabled).toBe(false); for (const s of a.defaultSources) { for (const k of Object.keys(s.config)) expect(a.configFields.map((f) => f.key)).toContain(k); - expect(s.slug.endsWith('-pages') || s.slug === 'sites-pasted').toBe(true); + expect(s.slug.endsWith('-pages') || ['sites-pasted', 'pit-crawler'].includes(s.slug)).toBe( + true, + ); } const all = ADAPTERS.flatMap((x) => (x.defaultSources ?? []).map((s) => s.slug)); expect(new Set(all).size).toBe(all.length); @@ -431,7 +456,9 @@ describe('a list of addresses', () => { return new Response(''); }, }; - const r = await readUrl('http://169.254.169.254/latest/meta-data', { http }); + const r = await readUrl('http://169.254.169.254/latest/meta-data', { + http, + }); expect(r.status).toBe('blocked'); expect(asked).toEqual([]); });