diff --git a/apps/cli/package.json b/apps/cli/package.json index c9c04d5..b37d9a2 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/nichedb", - "version": "0.57.0", + "version": "0.58.0", "description": "CLI and MCP bridge for NicheDB: browse collections, manage sources and feeds, search items, from any deployment", "type": "module", "main": "src/index.js", diff --git a/apps/cli/src/index.js b/apps/cli/src/index.js index 5f56596..19267d0 100644 --- a/apps/cli/src/index.js +++ b/apps/cli/src/index.js @@ -15,7 +15,7 @@ import { homedir, tmpdir } from 'node:os'; import { join } from 'node:path'; import { createInterface } from 'node:readline'; -export const VERSION = '0.57.0'; +export const VERSION = '0.58.0'; const GEO_FLAGS = ['lat', 'long', 'radius', 'unit', 'bbox', 'sort', 'offset']; const DEFAULT_API = process.env.NICHEDB_API ?? 'https://nichedb.dev'; const CONFIG_DIR = join(process.env.XDG_CONFIG_HOME ?? join(homedir(), '.config'), 'nichedb'); diff --git a/apps/web/package.json b/apps/web/package.json index 93d653f..27019f7 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/web", - "version": "0.57.0", + "version": "0.58.0", "private": true, "type": "module", "scripts": { diff --git a/apps/web/src/lib/sites.js b/apps/web/src/lib/sites.js index 8429079..f7148c6 100644 --- a/apps/web/src/lib/sites.js +++ b/apps/web/src/lib/sites.js @@ -179,6 +179,15 @@ const words = (list, max, len = 40) => .filter(Boolean) .slice(0, max); +const positive = (v) => (Number.isInteger(v) && v > 0 && v < 100_000 ? v : null); +/** A list of media references, each with a public address, at most `max`. */ +const mediaRefs = (list, max, shape) => + (Array.isArray(list) ? list : []) + .filter((m) => m && typeof m === 'object') + .map(shape) + .filter((m) => m.url) + .slice(0, max); + /** * One pushed record as the row it becomes, or the reason it is refused. * @@ -225,6 +234,19 @@ export function crawledItem(entry, { now = new Date() } = {}) { rendered: enrichment.rendered === true, depth: Number.isFinite(enrichment.depth) ? enrichment.depth : null, crawler: clip(enrichment.crawler, 80) || null, + // What the page shows, by reference (the crawler never stores the files). + images: mediaRefs(enrichment.images, 20, (m) => ({ + url: publicWebUrl(m.url), + alt: clip(m.alt, 300), + width: positive(m.width), + height: positive(m.height), + })), + videos: mediaRefs(enrichment.videos, 10, (m) => ({ + url: publicWebUrl(m.url), + title: clip(m.title, 300), + thumbnail: m.thumbnail ? publicWebUrl(m.thumbnail) : null, + provider: clip(m.provider, 40) || null, + })), }; const row = normaliseItem({ ...item, diff --git a/apps/web/src/routes/sites.test.js b/apps/web/src/routes/sites.test.js index a4ca599..e810ba9 100644 --- a/apps/web/src/routes/sites.test.js +++ b/apps/web/src/routes/sites.test.js @@ -320,6 +320,19 @@ describe('records a crawler pushes', () => { rendered: true, hash: 'abc123', crawler: 'pitbot/0.1', + images: [ + { url: 'https://crawled.example/dome.jpg', alt: 'Brick dome', width: 800, height: -1 }, + { url: 'http://127.0.0.1/secret.png', alt: 'private' }, + 'not an object', + ], + videos: [ + { + url: 'https://www.youtube.com/watch?v=dQw4w9WgXcQ', + title: 'Firing it', + thumbnail: 'https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg', + provider: 'youtube', + }, + ], }, }; @@ -352,6 +365,17 @@ describe('records a crawler pushes', () => { expect(row.tags).toContain('rendered'); expect(row.contentHash).toBe('crawl:abc123'); expect(row.data.enrichment.words).toBe(812); + expect(row.data.enrichment.images).toEqual([ + { url: 'https://crawled.example/dome.jpg', alt: 'Brick dome', width: 800, height: null }, + ]); + expect(row.data.enrichment.videos).toEqual([ + { + url: 'https://www.youtube.com/watch?v=dQw4w9WgXcQ', + title: 'Firing it', + thumbnail: 'https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg', + provider: 'youtube', + }, + ]); }); test('an empty or oversized batch is refused whole', async () => { diff --git a/apps/worker/package.json b/apps/worker/package.json index 69cc0ee..3e0a783 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/worker", - "version": "0.57.0", + "version": "0.58.0", "private": true, "type": "module", "scripts": { diff --git a/package.json b/package.json index a3ff022..12ad195 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "niche-db", - "version": "0.57.0", + "version": "0.58.0", "private": true, "type": "module", "description": "An open platform for large, ever-growing databases of real-time data: sources in, feeds out. Web, API, CLI and MCP on one Postgres.", diff --git a/packages/adapters/package.json b/packages/adapters/package.json index 668a7c7..525e32e 100644 --- a/packages/adapters/package.json +++ b/packages/adapters/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/adapters", - "version": "0.57.0", + "version": "0.58.0", "private": true, "type": "module", "exports": { diff --git a/packages/auth/package.json b/packages/auth/package.json index 02b90a0..baabdfd 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/auth", - "version": "0.57.0", + "version": "0.58.0", "private": true, "type": "module", "exports": { diff --git a/packages/config/package.json b/packages/config/package.json index bba602f..465aad1 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/config", - "version": "0.57.0", + "version": "0.58.0", "private": true, "type": "module", "exports": { diff --git a/packages/core/package.json b/packages/core/package.json index 67127bf..2a546fc 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/core", - "version": "0.57.0", + "version": "0.58.0", "private": true, "type": "module", "exports": { diff --git a/packages/db/package.json b/packages/db/package.json index 3711f71..8f3b418 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/db", - "version": "0.57.0", + "version": "0.58.0", "private": true, "type": "module", "exports": { diff --git a/packages/enrichers/package.json b/packages/enrichers/package.json index 0e78bf3..f6118e1 100644 --- a/packages/enrichers/package.json +++ b/packages/enrichers/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/enrichers", - "version": "0.57.0", + "version": "0.58.0", "private": true, "type": "module", "exports": { diff --git a/packages/knowledge/package.json b/packages/knowledge/package.json index c770d08..8551c5d 100644 --- a/packages/knowledge/package.json +++ b/packages/knowledge/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/knowledge", - "version": "0.57.0", + "version": "0.58.0", "private": true, "type": "module", "exports": { diff --git a/packages/notify/package.json b/packages/notify/package.json index 493c4c2..2fe0a91 100644 --- a/packages/notify/package.json +++ b/packages/notify/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/notify", - "version": "0.57.0", + "version": "0.58.0", "private": true, "type": "module", "exports": { diff --git a/packages/payments/package.json b/packages/payments/package.json index 265dd74..62ecc0f 100644 --- a/packages/payments/package.json +++ b/packages/payments/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/payments", - "version": "0.57.0", + "version": "0.58.0", "private": true, "type": "module", "exports": { diff --git a/packages/premium/package.json b/packages/premium/package.json index d4252f8..a7edbd5 100644 --- a/packages/premium/package.json +++ b/packages/premium/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/premium", - "version": "0.57.0", + "version": "0.58.0", "private": true, "type": "module", "exports": { diff --git a/packages/queue/package.json b/packages/queue/package.json index 78fd647..8abc3e3 100644 --- a/packages/queue/package.json +++ b/packages/queue/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/queue", - "version": "0.57.0", + "version": "0.58.0", "private": true, "type": "module", "exports": {