Sites: POST /api/v1/sites/records for records a crawler already read (0.57.0)

apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/nichedb",
-  "version": "0.56.0",
+  "version": "0.57.0",
   "description": "CLI and MCP bridge for NicheDB: browse collections, manage sources and feeds, search items, from any deployment",
   "type": "module",
   "main": "src/index.js",

apps/cli/src/index.js

@@ -15,7 +15,7 @@ import { homedir, tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { createInterface } from 'node:readline';
 
-export const VERSION = '0.56.0';
+export const VERSION = '0.57.0';
 const GEO_FLAGS = ['lat', 'long', 'radius', 'unit', 'bbox', 'sort', 'offset'];
 const DEFAULT_API = process.env.NICHEDB_API ?? 'https://nichedb.dev';
 const CONFIG_DIR = join(process.env.XDG_CONFIG_HOME ?? join(homedir(), '.config'), 'nichedb');

apps/web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nichedb/web",
-  "version": "0.56.0",
+  "version": "0.57.0",
   "private": true,
   "type": "module",
   "scripts": {

apps/web/src/lib/sites.js

@@ -3,16 +3,19 @@ import { normaliseItem } from '@nichedb/core/adapter';
 import { makeHttp } from '@nichedb/core/http';
 import {
   AGENT,
+  fit,
+  KINDS,
   LIST_MAX,
   parseUrlList,
+  publicWebUrl,
   readUrl,
   recordItem,
   recordPath,
   urlsForPath,
   webUrl,
 } from '@nichedb/core/opensite';
 import * as q from '@nichedb/db/queries';
-import { canAddSources, Denied } from './service.js';
+import { canAddSources, Denied, isAdmin } from './service.js';
 
 /**
  * The operations behind /c/sites, shared by the page, the API and the MCP
@@ -26,6 +29,11 @@ import { canAddSources, Denied } from './service.js';
 
 export const COLLECTION = 'sites';
 export const PASTED_SOURCE = 'sites-pasted';
+/** Where records a crawler read itself land: POST /api/v1/sites/records. */
+export const CRAWLER_SOURCE = 'pit-crawler';
+/** Records per push. A crawler sends batches; a bigger body is refused, not cut. */
+export const RECORDS_MAX = 200;
+const STATUSES = new Set(['live', 'gone', 'moved', 'blocked']);
 
 /** How stale a kept record may be before an ask reads the page again. */
 export const FRESH_MS = 60 * 60 * 1000;
@@ -136,8 +144,15 @@ export async function readAndKeep(url, { fetcher = null } = {}) {
   const record = await readUrl(asked, { http: fetcher ?? http() });
   if (!record) throw new Denied('That address could not be read.', 422);
   const item = normaliseItem(recordItem(record));
-  await q.upsertItems({ collectionId: collection.id, sourceId: source.id, items: [item] });
-  const kept = await q.itemByUrls({ collectionId: collection.id, urls: [record.canonical] });
+  await q.upsertItems({
+    collectionId: collection.id,
+    sourceId: source.id,
+    items: [item],
+  });
+  const kept = await q.itemByUrls({
+    collectionId: collection.id,
+    urls: [record.canonical],
+  });
   return { record, item: kept, path: pathOf(record) };
 }
 
@@ -153,3 +168,126 @@ export async function recordFor(url, { force = false, fetcher = null } = {}) {
   }
   return { ...(await readAndKeep(url, { fetcher })), fresh: false };
 }
+
+const clip = (value, max) => {
+  const text = typeof value === 'string' ? value.replace(/\s+/g, ' ').trim() : '';
+  return text.length > max ? `${text.slice(0, max - 1)}…` : text;
+};
+const words = (list, max, len = 40) =>
+  (Array.isArray(list) ? list : [])
+    .map((t) => clip(String(t ?? ''), len).toLowerCase())
+    .filter(Boolean)
+    .slice(0, max);
+
+/**
+ * One pushed record as the row it becomes, or the reason it is refused.
+ *
+ * The crawler read the page, so nothing is fetched here; what is checked is
+ * that the record is one this index would have drawn itself: a public web
+ * address, a known kind and status, within the spec's size. What the crawler
+ * learned on top (language, keywords, an excerpt of the text, links counted)
+ * rides along under `data.enrichment`, and the keywords become `topic:` tags
+ * and the excerpt the summary when the page has no description, because the
+ * search column is made of title, summary and tags.
+ */
+export function crawledItem(entry, { now = new Date() } = {}) {
+  const record = entry?.record ?? entry;
+  if (!record || typeof record !== 'object') return { error: 'not a record' };
+  const canonical = publicWebUrl(record.canonical ?? record.url);
+  if (!canonical) return { error: 'canonical is not a public web address' };
+  const url = publicWebUrl(record.url) ?? canonical;
+  const enrichment =
+    entry?.enrichment && typeof entry.enrichment === 'object' ? entry.enrichment : {};
+  const kept = fit({
+    ...record,
+    opensite: String(record.opensite ?? '0.1'),
+    url,
+    canonical,
+    kind: KINDS.includes(record.kind) ? record.kind : 'page',
+    title: clip(record.title, 300),
+    description: clip(record.description, 1000) || null,
+    status: STATUSES.has(record.status) ? record.status : 'live',
+    source: 'read',
+    tags: words(record.tags, 20),
+    fetched_at: Number.isNaN(Date.parse(record.fetched_at)) ? now.toISOString() : record.fetched_at,
+  });
+  const item = recordItem(kept);
+  if (!item) return { error: 'no canonical address' };
+  const keywords = words(enrichment.keywords, 20);
+  const language = clip(enrichment.language, 12).toLowerCase();
+  const excerpt = clip(enrichment.excerpt, 1000);
+  const extra = {
+    language: language || null,
+    keywords,
+    excerpt: excerpt || null,
+    words: Number.isFinite(enrichment.words) ? enrichment.words : null,
+    links: Number.isFinite(enrichment.links) ? enrichment.links : null,
+    rendered: enrichment.rendered === true,
+    depth: Number.isFinite(enrichment.depth) ? enrichment.depth : null,
+    crawler: clip(enrichment.crawler, 80) || null,
+  };
+  const row = normaliseItem({
+    ...item,
+    summary: item.summary || excerpt || null,
+    tags: [
+      ...item.tags,
+      language ? `lang:${language}` : null,
+      extra.rendered ? 'rendered' : null,
+      'crawled',
+      ...keywords.map((k) => `topic:${k}`),
+    ].filter((t, i, all) => t && all.indexOf(t) === i),
+    data: { ...item.data, enrichment: extra },
+  });
+  if (!row) return { error: 'no title or address' };
+  // The crawler's hash is of the page text, so a re-crawl of an unchanged page writes nothing
+  // (the upsert skips a row whose hash is the same); fetched_at alone is not a change.
+  const hash = clip(enrichment.hash, 128);
+  if (hash) row.contentHash = `crawl:${hash}`;
+  return { item: row };
+}
+
+/**
+ * Keep records a crawler already read: admins only (an admin API key is how a
+ * crawler signs in), at most RECORDS_MAX a call, each validated on its own so
+ * one bad record never sinks the batch. Upserted by canonical address into
+ * the crawler source, so a re-crawl updates the row and an unchanged page
+ * (same hash) writes nothing.
+ */
+export async function keepCrawled({ user, records }) {
+  if (!user) throw new Denied('Sign in with an API key to push records.', 401);
+  if (!isAdmin(user)) throw new Denied('Pushing crawled records is for the index operators.', 403);
+  if (!Array.isArray(records) || records.length === 0)
+    throw new Denied('send { "records": [ { "record": {…}, "enrichment": {…} } ] }', 400);
+  if (records.length > RECORDS_MAX)
+    throw new Denied(
+      `At most ${RECORDS_MAX} records a call; send ${records.length} as batches.`,
+      413,
+    );
+  const [collection, source] = await Promise.all([
+    q.getCollection(COLLECTION),
+    q.getSource(CRAWLER_SOURCE),
+  ]);
+  if (!collection || !source)
+    throw new Denied('The crawler source is not set up on this deployment yet.', 503);
+  const items = [];
+  const rejected = [];
+  records.forEach((entry, index) => {
+    const out = crawledItem(entry);
+    if (out.item) items.push(out.item);
+    else rejected.push({ index, error: out.error });
+  });
+  const { added, updated } = items.length
+    ? await q.upsertItems({
+        collectionId: collection.id,
+        sourceId: source.id,
+        items,
+      })
+    : { added: 0, updated: 0 };
+  return {
+    accepted: items.length,
+    added,
+    updated,
+    unchanged: items.length - added - updated,
+    rejected,
+  };
+}

apps/web/src/routes/sites.js

@@ -3,6 +3,7 @@ import { cached, render, respond } from '../lib/http.js';
 import { Denied } from '../lib/service.js';
 import {
   findByPath,
+  keepCrawled,
   listForHost,
   pathOf,
   readAndKeep,
@@ -68,7 +69,11 @@ export function registerSites(app) {
     if (pasted !== '' || file) {
       const text = `${pasted}\n${file ? await file.text() : ''}`;
       try {
-        const out = await submitBulk({ user: c.get('user'), text, name: String(body.name ?? '') });
+        const out = await submitBulk({
+          user: c.get('user'),
+          text,
+          name: String(body.name ?? ''),
+        });
         const notice = `${out.queued} addresses queued${out.rejected.length ? `, ${out.rejected.length} could not be read` : ''}${out.dropped ? `, ${out.dropped} over the cap of ten thousand left out` : ''}. The worker reads them a few hundred a minute; this page shows how far it has got.`;
         return c.redirect(`${out.page}?notice=${encodeURIComponent(notice)}`, 303);
       } catch (err) {
@@ -150,6 +155,24 @@ export function registerSites(app) {
     }
   });
 
+  /**
+   * Records a crawler already read (pit.world's spider), in batches. Admin
+   * API key only; see keepCrawled for what is checked.
+   */
+  app.post('/api/v1/sites/records', async (c) => {
+    const body = await c.req.json().catch(() => null);
+    try {
+      const out = await keepCrawled({
+        user: c.get('user'),
+        records: body?.records,
+      });
+      return c.json(out, 200);
+    } catch (err) {
+      if (!(err instanceof Denied)) throw err;
+      return c.json({ error: err.message }, err.status);
+    }
+  });
+
   app.get('/api/v1/sites', async (c) => {
     const url = (c.req.query('url') ?? '').trim();
     if (url === '') return c.json({ error: 'say which address: ?url=' }, 400);
@@ -186,7 +209,11 @@ export function registerSites(app) {
       );
     } catch (err) {
       if (!(err instanceof Denied)) throw err;
-      return respond(c, { error: err.message, status: err.status, redirectTo: '/c/sites/add' });
+      return respond(c, {
+        error: err.message,
+        status: err.status,
+        redirectTo: '/c/sites/add',
+      });
     }
   });