diff --git a/apps/poller/package.json b/apps/poller/package.json index 7fb85a7..1e85d1b 100644 --- a/apps/poller/package.json +++ b/apps/poller/package.json @@ -8,7 +8,7 @@ "test": "node --test test/*.test.js" }, "dependencies": { - "@profullstack/scraper-daemon": "0.2.2", + "@profullstack/scraper-daemon": "0.3.0", "@rssamplifier/db": "workspace:*", "@rssamplifier/discover": "workspace:*", "@rssamplifier/feed": "workspace:*", diff --git a/apps/poller/src/index.js b/apps/poller/src/index.js index ba2cabe..4b2027f 100644 --- a/apps/poller/src/index.js +++ b/apps/poller/src/index.js @@ -1022,6 +1022,11 @@ async function queueTick() { cards: backlogs.cardsPending, authors, }); + + // Due feeds per crawl tier and how late the oldest of each is: the line that + // says live feeds are being read on time while the quiet tiers drain. + const tiers = await q.dueByTier(db); + log('tiers', Object.fromEntries(tiers.map((t) => [t.tier, `${t.due} due, oldest ${t.oldest ?? '-'}`]))); } catch (err) { // A missed sample is a gap in a chart, which the chart draws as a gap. It // is not a reason to make noise in a log that is read for crawl failures. diff --git a/apps/web/src/app/RefreshFeed.jsx b/apps/web/src/app/RefreshFeed.jsx new file mode 100644 index 0000000..c792efa --- /dev/null +++ b/apps/web/src/app/RefreshFeed.jsx @@ -0,0 +1,54 @@ +'use client'; + +import { useRouter } from 'next/navigation'; +import { useState } from 'react'; + +const SAID = { + recent: () => 'Already read in the last few minutes.', + read: (r) => (r.newItems > 0 ? `${r.newItems} new ${r.newItems === 1 ? 'post' : 'posts'}.` : 'Read just now: nothing new.'), + unchanged: () => 'Read just now: nothing new.', + queued: () => 'Queued: this source is read by the crawler, first in line.', + reading: () => 'Still reading; this page will catch up shortly.', + throttled: () => 'The publisher asked us to slow down; try again later.', + failed: (r) => `Could not read it: ${r.error ?? 'the publisher did not answer'}.`, + dead: (r) => r.error, +}; + +/** + * Read this feed now instead of waiting for its turn. POSTs to + * /api/feeds//refresh, which runs the crawler's own read, then re-renders + * the page so new posts and the freshness line show up in place. + * + * @param {{ slug: string }} props + */ +export default function RefreshFeed({ slug }) { + const router = useRouter(); + const [busy, setBusy] = useState(false); + const [said, setSaid] = useState(''); + + async function refresh() { + setBusy(true); + setSaid('Reading the feed…'); + try { + const res = await fetch(`/api/feeds/${encodeURIComponent(slug)}/refresh`, { method: 'POST' }); + const body = await res.json().catch(() => ({})); + setSaid((SAID[body.state] ?? SAID.failed)(body)); + if (body.ok) router.refresh(); + } catch { + setSaid('Could not reach the server.'); + } finally { + setBusy(false); + } + } + + return ( +

+ + + {said} + +

+ ); +} diff --git a/apps/web/src/app/[slug]/page.jsx b/apps/web/src/app/[slug]/page.jsx index 11a282a..8dd7573 100644 --- a/apps/web/src/app/[slug]/page.jsx +++ b/apps/web/src/app/[slug]/page.jsx @@ -23,6 +23,7 @@ import FollowControls from '../FollowControls.jsx'; import ListFilter from '../ListFilter.jsx'; import { FILTER_FROM } from '../../lib/listFilter.js'; import Freshness from '../Freshness.jsx'; +import RefreshFeed from '../RefreshFeed.jsx'; import PlayButton from '../PlayButton.jsx'; import QueueAll from '../QueueAll.jsx'; import QueueButton from '../QueueButton.jsx'; @@ -326,6 +327,7 @@ export default async function FeedPage({ params, base }) { lib/freshness.js. The newest post's date comes off the list this page has already loaded, so the signal costs no extra query. */} + {/* Follow and share, side by side, because they are the two things to do with a feed you have just found and only one of them needs an diff --git a/apps/web/src/app/api/feeds/[slug]/refresh/route.js b/apps/web/src/app/api/feeds/[slug]/refresh/route.js new file mode 100644 index 0000000..78d266a --- /dev/null +++ b/apps/web/src/app/api/feeds/[slug]/refresh/route.js @@ -0,0 +1,82 @@ +import { q } from '@rssamplifier/db'; +import { crawlFeed } from '@rssamplifier/ingest'; +import { isCollected } from '@rssamplifier/social'; + +import { db } from '../../../../../lib/db.js'; + +export const dynamic = 'force-dynamic'; + +/** A feed read this recently is answered from what we have: a click is not a reason to ask the publisher twice. */ +const RECENT_MS = 5 * 60_000; + +/** How long a click waits on the publisher before it is told the read carries on without it. */ +const WAIT_MS = 25_000; + +/** Reads this worker has in flight, so a double click is one request to the publisher. */ +const inflight = new Map(); + +const json = (body, status = 200) => + new Response(JSON.stringify(body), { + status, + headers: { + 'content-type': 'application/json; charset=utf-8', + 'cache-control': 'no-store', + 'access-control-allow-origin': '*', + }, + }); + +/** + * Read one feed now: the refresh button on a feed page (and anyone's agent). + * + * The crawler reads every feed on its own schedule, but a schedule is a guess + * and a reader looking at a stale page knows better. This runs the crawler's + * own crawlFeed in the request, so what lands is exactly what a scheduled read + * would have stored, and the cadence it writes is recomputed from the fresh + * document. Social sources are collected by the poller's runtime, not here, so + * those are moved to the front of their queue instead. + * + * @param {Request} _req + * @param {{ params: Promise<{ slug: string }> }} ctx + */ +export async function POST(_req, { params }) { + const { slug } = await params; + const client = db(); + const feed = await q.feedBySlug(client, slug); + if (!feed) return json({ error: 'not-found' }, 404); + if (feed.status === 'dead') return json({ ok: false, state: 'dead', error: 'This feed has been retired after repeated failures.' }, 409); + + const last = Date.parse(String(feed.last_fetched_at ?? '')); + if (Number.isFinite(last) && Date.now() - last < RECENT_MS) { + return json({ ok: true, state: 'recent', checkedAt: new Date(last).toISOString(), newItems: 0 }); + } + + if (isCollected(feed)) { + await q.requestRefresh(client, String(feed.id)); + return json({ ok: true, state: 'queued' }, 202); + } + + const id = String(feed.id); + let run = inflight.get(id); + if (!run) { + run = crawlFeed(client, feed) + .catch((err) => ({ ok: false, error: String(err?.message ?? err) })) + .finally(() => inflight.delete(id)); + inflight.set(id, run); + } + + const res = await Promise.race([run, new Promise((resolve) => setTimeout(() => resolve(null), WAIT_MS))]); + if (res === null) return json({ ok: true, state: 'reading' }, 202); + if (!res.ok) { + return json({ ok: false, state: res.throttled ? 'throttled' : 'failed', error: res.error ?? 'The publisher did not answer.' }, 502); + } + + const after = await q.feedBySlug(client, slug); + return json({ + ok: true, + state: res.notModified ? 'unchanged' : 'read', + newItems: Number(res.newItems ?? 0), + checkedAt: after?.last_success_at ?? null, + lastPublishedAt: after?.last_published_at ?? null, + nextCheckAt: after?.next_fetch_at ?? null, + }); +} diff --git a/apps/web/src/app/api/feeds/[slug]/route.js b/apps/web/src/app/api/feeds/[slug]/route.js index c1dd288..d0cf3fb 100644 --- a/apps/web/src/app/api/feeds/[slug]/route.js +++ b/apps/web/src/app/api/feeds/[slug]/route.js @@ -62,6 +62,7 @@ export async function GET(req, { params }) { freshness: fresh.state, lastPublishedAt: fresh.publishedAt, nextCheckAt: fresh.nextCheckAt, + archived: fresh.archived, page: `${siteUrl()}/${feed.slug}`, topics: topics.map((t) => ({ slug: t.slug, diff --git a/apps/web/src/app/globals.css b/apps/web/src/app/globals.css index 0a3c397..32107fe 100644 --- a/apps/web/src/app/globals.css +++ b/apps/web/src/app/globals.css @@ -3932,6 +3932,36 @@ html:not([data-dock]) .playlist-hint { border-bottom: 1px dotted var(--line); } +.refresh-feed { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 0.6rem; + margin: 0.5rem 0 0; + font-family: var(--mono); + font-size: 0.74rem; + color: var(--muted); +} + +.refresh-feed-button { + font: inherit; + cursor: pointer; + padding: 0.3rem 0.7rem; + border: 1px solid var(--line); + border-radius: 4px; + background: transparent; + color: var(--fg); +} + +.refresh-feed-button:hover:not(:disabled) { + border-color: var(--accent); +} + +.refresh-feed-button:disabled { + cursor: progress; + opacity: 0.6; +} + .freshness-note { display: inline-block; margin-top: 0.35rem; diff --git a/apps/web/src/lib/freshness.js b/apps/web/src/lib/freshness.js index 5d799ac..90d783f 100644 --- a/apps/web/src/lib/freshness.js +++ b/apps/web/src/lib/freshness.js @@ -45,6 +45,12 @@ const OVERDUE_MULTIPLE = 2; /** Below this, a feed that has not published is called dormant rather than quiet. */ const DORMANT_DAYS = 365; +/** Past a week between reads, a feed is archived (see @profullstack/scraper-daemon/cadence). */ +const ARCHIVED_AFTER_INTERVAL = 7 * DAY; + +/** ...and only once it has been silent this long. */ +const ARCHIVED_AFTER_SILENCE = 180 * DAY; + /** * A short, plain interval. "3 minutes", "4 hours", "2 days", "8 months". * @@ -115,6 +121,10 @@ export function freshness(feed = {}, newestPost = null, now = Date.now()) { publishedAt: published === null ? null : new Date(published).toISOString(), publishedGap: published === null ? null : humanGap(now - published), nextCheckAt: next === null ? null : new Date(next).toISOString(), + // Archived: silent six months, so the crawler reads it once a quarter rather + // than with the live feeds. The house cadence marks that with an interval + // past a week (the longest a live feed ever waits), so the interval says it. + archived: interval > ARCHIVED_AFTER_INTERVAL && published !== null && now - published > ARCHIVED_AFTER_SILENCE, }; // Never successfully read. Distinct from every other state: we are not @@ -153,6 +163,14 @@ export function freshness(feed = {}, newestPost = null, now = Date.now()) { // Read on time, but the publisher has gone quiet. This is the one the reader // most needs and is least likely to guess: the data is current, and current // means "nothing since 2023". + if (base.archived) { + return { + ...base, + state: 'dormant', + note: `Archived: nothing published for ${humanGap(now - published)}. Kept for the record and checked once a quarter in case it returns.`, + }; + } + if (published !== null && now - published > DORMANT_DAYS * DAY) { return { ...base, diff --git a/apps/web/test/freshness.test.js b/apps/web/test/freshness.test.js index 7e37c9c..0c242c1 100644 --- a/apps/web/test/freshness.test.js +++ b/apps/web/test/freshness.test.js @@ -146,3 +146,12 @@ test('every state has a label, so the badge can never render undefined', () => { for (const s of reached) assert.ok(states.includes(s), `${s} is a known state`); assert.equal(reached.size, states.length, 'all five states are reachable'); }); + +test('an archived feed reads dormant and says it is checked once a quarter', () => { + const archived = { ...live, last_success_at: iso(30 * DAY), fetch_interval_minutes: 90 * 24 * 60 }; + const f = freshness(archived, iso(200 * DAY), NOW); + assert.equal(f.archived, true); + assert.equal(f.state, 'dormant'); + assert.match(f.note, /once a quarter/); + assert.equal(freshness({ ...archived, fetch_interval_minutes: 60 }, iso(200 * DAY), NOW).archived, false); +}); diff --git a/bun.lock b/bun.lock index ad68a47..a0de009 100644 --- a/bun.lock +++ b/bun.lock @@ -22,7 +22,7 @@ "name": "@rssamplifier/poller", "version": "0.1.0", "dependencies": { - "@profullstack/scraper-daemon": "0.2.2", + "@profullstack/scraper-daemon": "0.3.0", "@rssamplifier/db": "workspace:*", "@rssamplifier/discover": "workspace:*", "@rssamplifier/feed": "workspace:*", @@ -87,6 +87,7 @@ "version": "0.1.0", "dependencies": { "@libsql/client": "^0.18.0", + "@profullstack/scraper-daemon": "0.3.0", "@rssamplifier/feed": "workspace:*", "bullmq": "^6.4.2", "ioredis": "^6.0.0", @@ -118,6 +119,7 @@ "name": "@rssamplifier/ingest", "version": "0.1.0", "dependencies": { + "@profullstack/scraper-daemon": "0.3.0", "@rssamplifier/db": "workspace:*", "@rssamplifier/feed": "workspace:*", "@rssamplifier/mail": "workspace:*", @@ -370,7 +372,7 @@ "@profullstack/rssamplifier": ["@profullstack/rssamplifier@workspace:apps/cli"], - "@profullstack/scraper-daemon": ["@profullstack/scraper-daemon@0.2.2", "", { "peerDependencies": { "bullmq": ">=6", "ioredis": ">=5" }, "optionalPeers": ["bullmq", "ioredis"], "bin": { "scraper-daemon": "bin/scraper-daemon.js" } }, "sha512-V/+WT7q6VOedkkepYszctkXdfvWGZbmEZMi9Q7nejO35pg8kPVwR0W53GjswdsAXM/AqA+wMCfB5/oQo1Ihifw=="], + "@profullstack/scraper-daemon": ["@profullstack/scraper-daemon@0.3.0", "", { "peerDependencies": { "bullmq": ">=6", "ioredis": ">=5" }, "optionalPeers": ["bullmq", "ioredis"], "bin": { "scraper-daemon": "bin/scraper-daemon.js" } }, "sha512-C/COc8Oef+/8/Bdu/uyvTZbeIO0rMm3q7wEJNzDqcuoZ+cEs6OEGzrKwCkA1N7zqyi3NASCAytfeFxJiiaG9TQ=="], "@profullstack/sealed-secrets": ["@profullstack/sealed-secrets@0.1.0", "", {}, "sha512-6LRa+yUT7PQ5tZNoYEnf3VVxsyw/dqHqAtYpk/aS/bntxf3mQH+2MwNjQoBd7fCqdN97TvZup8qhrpfUcZlfng=="], diff --git a/packages/db/migrations-pg/20261029010615_feeds_due_tier_idx.sql b/packages/db/migrations-pg/20261029010615_feeds_due_tier_idx.sql new file mode 100644 index 0000000..e36f80d --- /dev/null +++ b/packages/db/migrations-pg/20261029010615_feeds_due_tier_idx.sql @@ -0,0 +1,12 @@ +-- The crawl tier, as the dispatcher reads it: due feeds per tier, oldest due +-- first (queries.js dueFeeds / TIER_SQL). The expression must stay exactly what +-- tierSql('fetch_interval_minutes') from @profullstack/scraper-daemon/cadence +-- returns, or the planner will not use this index. +-- +-- Concurrently, because feeds is read by every page: migrate() runs each +-- statement on its own with no transaction around the file, which is what +-- create index concurrently needs. Built by hand on production first, so on +-- boot this is a no-op. +create index concurrently if not exists feeds_due_tier_idx + on feeds ((case when fetch_interval_minutes <= 60 then 0 when fetch_interval_minutes <= 1440 then 1 when fetch_interval_minutes < 10080 then 2 when fetch_interval_minutes <= 10080 then 3 else 4 end), next_fetch_at) + where status <> 'dead'; diff --git a/packages/db/package.json b/packages/db/package.json index 8f76c92..751bbff 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -24,6 +24,7 @@ "bullmq": "^6.4.2", "ioredis": "^6.0.0", "pg": "^8.23.1", - "pg-copy-streams": "^7.0.0" + "pg-copy-streams": "^7.0.0", + "@profullstack/scraper-daemon": "0.3.0" } } diff --git a/packages/db/src/queries.js b/packages/db/src/queries.js index 7cf3340..f88c63a 100644 --- a/packages/db/src/queries.js +++ b/packages/db/src/queries.js @@ -5,6 +5,7 @@ import { clusterKey, dedupeItems, topicSlug } from '@rssamplifier/feed'; import { newId, nowIso } from './client.js'; import { FeedRemovedError, dropRemoved, isRemovedUrl, removalHost } from './removals.js'; import { topicLabelSql } from './topicLabel.js'; +import { quotas, tierSql, TIERS } from '@profullstack/scraper-daemon/cadence'; /** * Every query the app needs, in one place. @@ -2697,7 +2698,30 @@ export async function expressFeeds(db, limit = 25) { } /** - * Feeds whose next_fetch_at has passed, hand-submitted ones first. + * A feed's crawl tier, from its interval alone: 0 hot (read at least hourly), + * 1 active (daily), 2 slow, 3 dormant (gone quiet, read weekly), 4 archived + * (silent six months, read once a quarter). The house cadence + * (@profullstack/scraper-daemon/cadence) sets the interval from how often the + * feed posts, so the interval already says how much a feed is worth reading + * and a tier needs no column of its own: `feeds_due_tier_idx` is built on this + * exact expression, which is why it must stay byte-for-byte what tierSql() says. + */ +export const TIER_SQL = tierSql('fetch_interval_minutes'); + +/** + * Feeds whose next_fetch_at has passed, hand-submitted ones first, then by tier. + * + * Oldest-due-first was the whole reason a live weekly show could sit a week + * stale: after the crawler stalled for four days (2026-10-24..28) 260k feeds + * were due at once, and a show that came due on the 25th waited behind every + * dormant blog that came due on the 24th. Now each tier gets its share of the + * batch (40% hot, 30% active, 18% slow, 10% dormant, 2% archived), slots a tier + * cannot fill are handed down and then back up, and within a tier it is still + * oldest due first. Under load the quiet tiers lag, never the live ones; with + * room to spare everything due is read. Archived feeds only ever get their 2% + * (or what nothing else wanted), so the queue works off live feeds. + * + * One range read per tier on feeds_due_tier_idx. * * @param {Client} db * @param {number} [limit] @@ -2707,41 +2731,85 @@ export async function dueFeeds(db, limit = 25) { // Bounded rather than unbounded even though the express table is tiny: the // point of a reserved share is that it is reserved in both directions. const express = await expressFeeds(db, Math.floor(limit * EXPRESS_SHARE)); - const remaining = limit - express.length; - if (remaining <= 0) return express; + const room = limit - express.length; + if (room <= 0) return express; - const { rows } = await db.execute({ - // slug and title are along for the log: a crawler log line that names the - // blog and links to its page is worth two columns the crawl itself is - // holding the row for anyway. source_kind is what main added, and the - // crawl still needs it to know what it is fetching. - // item_count comes along so a crawl that stored nothing can pass the number - // straight back instead of paying for a count(*) to be told it is unchanged. - // - // last_published_at is what the crawl compares the document against to - // decide whether this publisher has published since we last looked -- the - // guard that keeps topics and credits from being rewritten on every crawl of - // an unchanged feed. It was missing from this select, so `knownPublished` - // read null for every feed the poller crawled and the guard was open in - // production the whole time. - // - // The last four are the conditional-request and change-detection signals - // added in 0032: what the server's validators were, what the feed contained, - // and when we last saw that change. They are what lets a feed stating no - // dates be scheduled on evidence rather than on the doubling ladder. - // + const now = nowIso(); + const out = [...express]; + const exhausted = new Set(); + const seen = new Set(express.map((r) => String(r.id))); + + const take = async (tier, want) => { // The express rows are excluded rather than deduplicated afterwards: they // have just been read and are about to be crawled, and handing the same // feed to two workers in one tick is two crawls of it. - sql: `select ${DUE_COLUMNS} - from feeds - where status <> 'dead' and next_fetch_at <= ? - and not (priority > 0 and last_fetched_at is null) - order by next_fetch_at asc limit ?`, - args: [nowIso(), remaining], + const { rows } = await db.execute({ + sql: `select ${DUE_COLUMNS} + from feeds + where status <> 'dead' and ${TIER_SQL} = ? and next_fetch_at <= ? + and not (priority > 0 and last_fetched_at is null) + order by next_fetch_at asc limit ?`, + args: [TIERS.indexOf(tier), now, want + seen.size], + }); + let got = 0; + for (const row of rows) { + if (got >= want) break; + if (seen.has(String(row.id))) continue; + seen.add(String(row.id)); + out.push(row); + got += 1; + } + if (got < want) exhausted.add(tier); + return got; + }; + + let carry = 0; + for (const [tier, quota] of quotas(room)) { + const want = quota + carry; + carry = want > 0 ? want - (await take(tier, want)) : 0; + } + for (const tier of TIERS) { + if (carry <= 0) break; + if (!exhausted.has(tier)) carry -= await take(tier, carry); + } + return out; +} + +/** + * Put a feed at the front of its tier's queue: the refresh button, for a feed + * the web process cannot read itself (a social source collected by the poller). + * The epoch rather than now(), because a tier is drained oldest due first and + * a backlog can be days deep; the express lane is for first crawls only. + * + * @param {Client} db + * @param {string} id + */ +export async function requestRefresh(db, id) { + await db.execute({ + sql: `update feeds set next_fetch_at = '1970-01-01T00:00:00.000Z' where id = ? and status <> 'dead'`, + args: [id], }); +} - return express.concat(rows); +/** + * Due feeds per tier, and how late the oldest of each is: the number that says + * whether live feeds are being read on time even while the backlog drains. + * + * @param {Client} db + * @returns {Promise>} + */ +export async function dueByTier(db) { + const now = nowIso(); + const out = []; + for (const [i, tier] of TIERS.entries()) { + const { rows } = await db.execute({ + sql: `select count(*) as due, min(next_fetch_at) as oldest + from feeds where status <> 'dead' and ${TIER_SQL} = ? and next_fetch_at <= ?`, + args: [i, now], + }); + out.push({ tier, due: Number(rows[0]?.due ?? 0), oldest: rows[0]?.oldest ?? null }); + } + return out; } /** diff --git a/packages/db/test/due-tiers.test.js b/packages/db/test/due-tiers.test.js new file mode 100644 index 0000000..4dcc4d2 --- /dev/null +++ b/packages/db/test/due-tiers.test.js @@ -0,0 +1,73 @@ +import assert from 'node:assert/strict'; +import { test, before, after } from 'node:test'; + +import { connectTest } from '../src/testdb.js'; +import * as q from '../src/queries.js'; + +let db; + +const DAY = 24 * 60 * 60_000; +const ago = (ms) => new Date(Date.now() - ms).toISOString(); + +before(async () => { + db = await connectTest(); +}); + +after(async () => { + db.close(); +}); + +/** `n` crawled feeds read every `minutes`, all due since `dueAgo` ms. */ +async function seed(prefix, n, minutes, dueAgo) { + await q.insertFeedsBulk( + db, + Array.from({ length: n }, (_, i) => ({ + slug: `${prefix}-${i}`, + feed_url: `https://${prefix}-${i}.example/feed.xml`, + title: `${prefix} ${i}`, + next_fetch_at: ago(dueAgo), + })), + ); + await db.execute({ + sql: `update feeds set fetch_interval_minutes = ?, last_fetched_at = ? where slug like ?`, + args: [minutes, ago(dueAgo + DAY), `${prefix}-%`], + }); +} + +test('a live feed due today is read before a dormant backlog due four days ago', async () => { + // Production on 2026-10-29: the crawler had stalled 10-24..28, 260k feeds + // were due, and a weekly show due on the 25th waited behind every quiet blog + // due on the 24th because the queue was oldest-due-first. + await seed('dormant', 40, 10_080, 4 * DAY); + await seed('archived', 40, 129_600, 5 * DAY); + await seed('weekly-show', 1, 360, DAY / 24); + + const due = await q.dueFeeds(db, 10); + const slugs = due.map((r) => String(r.slug)); + + assert.equal(slugs[0], 'weekly-show-0', 'the live feed comes first'); + assert.equal(due.length, 10, 'and the rest of the batch still drains the backlog'); + assert.ok(slugs.filter((s) => s.startsWith('archived')).length <= 1, 'archived feeds only trickle while others are due'); +}); + +test('with nothing live due, the batch is still filled from the quiet tiers', async () => { + await db.execute({ sql: `update feeds set next_fetch_at = ? where slug like 'weekly-show-%'`, args: [new Date(Date.now() + DAY).toISOString()] }); + const due = await q.dueFeeds(db, 30); + assert.equal(due.length, 30); + assert.equal(new Set(due.map((r) => r.slug)).size, 30, 'no feed twice'); +}); + +test('dueByTier reports each tier', async () => { + const tiers = await q.dueByTier(db); + assert.deepEqual(tiers.map((t) => t.tier), ['hot', 'active', 'slow', 'dormant', 'archived']); + assert.equal(tiers.find((t) => t.tier === 'dormant').due, 40); + assert.equal(tiers.find((t) => t.tier === 'archived').due, 40); +}); + +test('requestRefresh puts a feed at the front of its tier', async () => { + const [row] = (await db.execute({ sql: `select id from feeds where slug = 'dormant-39'`, args: [] })).rows; + await q.requestRefresh(db, String(row.id)); + await db.execute({ sql: `update feeds set next_fetch_at = ? where slug like 'archived-%'`, args: [new Date(Date.now() + DAY).toISOString()] }); + const due = await q.dueFeeds(db, 1); + assert.equal(due[0].slug, 'dormant-39'); +}); diff --git a/packages/ingest/package.json b/packages/ingest/package.json index 9256a8f..602e0c1 100644 --- a/packages/ingest/package.json +++ b/packages/ingest/package.json @@ -19,6 +19,7 @@ "@rssamplifier/db": "workspace:*", "@rssamplifier/mail": "workspace:*", "@rssamplifier/search": "workspace:*", - "@rssamplifier/social": "workspace:*" + "@rssamplifier/social": "workspace:*", + "@profullstack/scraper-daemon": "0.3.0" } } diff --git a/packages/ingest/src/cadence.js b/packages/ingest/src/cadence.js index 8b820ae..6edaa18 100644 --- a/packages/ingest/src/cadence.js +++ b/packages/ingest/src/cadence.js @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto'; +import { planNext } from '@profullstack/scraper-daemon/cadence'; /** * How often a feed is worth re-reading, judged by how often it actually posts. @@ -37,6 +38,16 @@ import { createHash } from 'node:crypto'; /** Never re-read a feed more often than this, however fast it posts. */ export const MIN_INTERVAL = 60; +/** + * The floor for a feed whose own dates say it posts more often than hourly. + * + * Dates are the publisher's evidence and the strongest we get, so a feed that + * demonstrably posts every half hour is read every fifteen minutes. Undated and + * change-log feeds keep MIN_INTERVAL: what we observe ourselves is only as fine + * as how often we looked, and they are 44% of the demand. + */ +export const DATED_MIN_INTERVAL = 15; + /** * The floor for a source we collect through a provider rather than fetch. * @@ -78,34 +89,6 @@ export const MAX_INTERVAL = 129_600; // 90 days */ const SAMPLE = 30; -/** - * How far past its own rhythm a feed must fall before it counts as having gone - * quiet, rather than merely being between posts. - * - * Three, because a weekly blog that misses a week is still a weekly blog, and - * one that has missed three has changed. Below this multiple the feed is - * scheduled on its rhythm; above it, on its silence. - */ -const QUIET_MULTIPLE = 3; - -/** - * The gap between consecutive posts, in minutes, as a sorted list. - * - * @param {number[]} times epoch ms, any order - * @returns {number[]} ascending gaps in minutes - */ -function gaps(times) { - const sorted = [...times].sort((a, b) => a - b); - const out = []; - for (let i = 1; i < sorted.length; i += 1) { - const gap = (sorted[i] - sorted[i - 1]) / 60_000; - // Two posts published in the same instant are one editorial act, not a - // rhythm of zero. Dropping them keeps a site that publishes its whole - // morning batch at 06:00 from being read as posting continuously. - if (gap > 0) out.push(gap); - } - return out.sort((a, b) => a - b); -} /** * The dates a document actually carries, cleaned. @@ -200,7 +183,8 @@ export function newestPublished(items, now = Date.now()) { export function intervalFromDates(items, now = Date.now(), floor = MIN_INTERVAL) { const times = publishedTimes(items, now); if (times.length < 2) return null; - return scheduleFrom(times, now, floor); + // A per-platform floor (a social source's) is a promise to that platform and stands. + return scheduleFrom(times, now, floor === MIN_INTERVAL ? DATED_MIN_INTERVAL : floor); } /** @@ -218,45 +202,15 @@ export function intervalFromDates(items, now = Date.now(), floor = MIN_INTERVAL) * @returns {number} minutes, between MIN_INTERVAL and MAX_INTERVAL */ function scheduleFrom(times, now, floor = MIN_INTERVAL) { - const silence = Math.max(0, (now - times[0]) / 60_000); - - // The typical gap, not the mean. A blog that posted forty times during one - // conference and monthly otherwise has a mean that describes neither; the - // median describes the ordinary week, which is what we are scheduling for. - const spacing = gaps(times); - - // Every instant is the same one, or there is only one of them. That is a feed - // with a single publishing event, not a feed with a rhythm of zero — an - // archive dumped in one go, or a generator that stamps every entry with the - // build time. There is no cadence to infer, so schedule it on its silence - // alone, which is the only real evidence available. - if (spacing.length === 0) return clamp(silence / 4, floor, MAX_INTERVAL); - - const rhythm = median(spacing); - - // Still keeping to its own rhythm: read it at that rhythm. Halved, so a post - // is found in the first half of its cycle rather than, on average, halfway - // through the next one — the directory's freshness promise is about how long - // a post can sit unseen, and this is the term that bounds it. - if (silence <= rhythm * QUIET_MULTIPLE) { - return clamp(rhythm / 2, floor, MAX_INTERVAL); - } - - // Gone quiet relative to its own history. Schedule on the silence instead, - // which makes the back-off self-scaling: the longer a feed stays dead the - // less often it is asked, without a table of thresholds to maintain and - // without ever quite giving up on it. - return clamp(silence / 4, floor, MAX_INTERVAL); -} - -/** - * @param {number[]} sorted ascending - * @returns {number} - */ -function median(sorted) { - if (sorted.length === 0) return MIN_INTERVAL; - const mid = Math.floor(sorted.length / 2); - return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; + // The policy itself is the house one, shared with p0dcasters: between posts a + // feed is read four times per rhythm but never sleeps past the point its next + // post is due; inside that window it is read 24 times per rhythm (at most six + // hours apart), so a weekly show is checked every few hours around release day + // instead of every 3.5 days. A feed silent for three rhythms backs off on its + // silence, capped at a week; silent 180 days and it is archived, read again in + // 90 (MAX_INTERVAL). The floor stays this module's, per source. + const plan = planNext(times, { now, floor, revisitEvery: MAX_INTERVAL }); + return plan ? plan.minutes : clamp(MIN_INTERVAL, floor, MAX_INTERVAL); } /** diff --git a/packages/ingest/test/cadence.test.js b/packages/ingest/test/cadence.test.js index e560471..4e34d4e 100644 --- a/packages/ingest/test/cadence.test.js +++ b/packages/ingest/test/cadence.test.js @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; -import { nextInterval, publishedTimes, MIN_INTERVAL, MAX_INTERVAL } from '../src/cadence.js'; +import { nextInterval, publishedTimes, MIN_INTERVAL, DATED_MIN_INTERVAL, MAX_INTERVAL } from '../src/cadence.js'; /** A fixed clock, so a test never depends on when it runs. */ const NOW = Date.parse('2026-08-18T12:00:00.000Z'); @@ -25,18 +25,25 @@ const hours = (minutes) => minutes / 60; const days = (minutes) => minutes / 1440; test('a feed keeping its own rhythm is read at that rhythm', () => { - // A weekly blog, posting on time. Half its cycle, so a post is found in the - // first half of the week rather than on average halfway through the next one. + // A weekly blog the day after a post: a quarter of its cycle. const weekly = nextInterval({ items: posts(10, 7, 1), now: NOW }); - assert.ok(days(weekly) > 3 && days(weekly) < 4, `weekly blog -> ${days(weekly).toFixed(1)}d`); + assert.equal(days(weekly), 1.75, `weekly blog -> ${days(weekly).toFixed(2)}d`); - // A daily paper. + // The same blog six days in, when the next post is due: every six hours, not + // every 3.5 days, which is what left weekly shows a week stale. + const due = nextInterval({ items: posts(10, 7, 6), now: NOW }); + assert.equal(hours(due), 6, `weekly blog on release day -> ${hours(due)}h`); + + // A daily paper, a few hours after its edition. const daily = nextInterval({ items: posts(20, 1, 0.2), now: NOW }); - assert.ok(hours(daily) > 11 && hours(daily) < 13, `daily paper -> ${hours(daily).toFixed(1)}h`); + assert.equal(hours(daily), 6, `daily paper -> ${hours(daily).toFixed(1)}h`); - // A news site publishing every couple of hours never goes below the floor. + // A news site publishing every couple of hours is read every half hour, and a + // firehose bottoms out at the dated floor, not below. const busy = nextInterval({ items: posts(20, 1 / 12, 0.01), now: NOW }); - assert.equal(busy, MIN_INTERVAL, 'a firehose is still only read hourly'); + assert.equal(busy, 30, `every-2h site -> ${busy}m`); + const firehose = nextInterval({ items: posts(20, 1 / 72, 0.001), now: NOW }); + assert.equal(firehose, DATED_MIN_INTERVAL, 'a firehose is read every fifteen minutes'); }); test('the abandoned podcast that started all this', () => { @@ -53,20 +60,21 @@ test('backing off is self-scaling, so silence needs no table of thresholds', () // Each of these is a weekly blog; only the silence differs. // Inside three times its own rhythm the feed is not late, it is between - // posts, so the interval does not move at all -- a weekly blog that misses a - // week is still a weekly blog. - assert.equal(at(3), at(21), 'a fortnight late is still on rhythm'); + // posts -- a weekly blog that misses a week is still a weekly blog, and one + // whose post is overdue is watched closely. + assert.equal(hours(at(21)), 6, 'a fortnight late is still on rhythm, and watched'); - const late = at(40); - const sixMonths = at(180); - const aYear = at(365); - const twoYears = at(730); + const late = at(25); + const quiet = at(40); + const halfYear = at(179); + const archived = at(181); assert.ok(at(21) < late, 'past three cycles, the gap starts widening'); - assert.ok(late < sixMonths, 'and it keeps widening'); - assert.ok(sixMonths < aYear, 'monotonically, with no cliff'); - assert.equal(aYear, MAX_INTERVAL, 'reaching the ceiling after about a year of silence'); - assert.equal(twoYears, MAX_INTERVAL, 'and never exceeding it'); + assert.ok(late < quiet, 'and it keeps widening'); + assert.equal(days(quiet), 7, 'up to a week, the longest a live feed waits'); + assert.equal(halfYear, quiet, 'and stays there'); + assert.equal(archived, MAX_INTERVAL, 'six months of silence archives it: read again in 90 days'); + assert.equal(at(730), MAX_INTERVAL, 'and never past that'); }); test('a feed that comes back is picked up again rather than written off', () => { @@ -139,8 +147,8 @@ test('posts sharing a timestamp are one editorial act, not a rhythm of zero', () } } const interval = nextInterval({ items: batched, now: NOW }); - assert.ok(interval >= MIN_INTERVAL, 'never below the floor'); - assert.ok(hours(interval) > 5, `same-instant posts do not read as continuous -> ${hours(interval).toFixed(1)}h`); + // A rhythm of a day, a day since the last batch: inside the window, so hourly. + assert.equal(interval, 60, `same-instant posts do not read as continuous -> ${hours(interval).toFixed(1)}h`); }); test('the result is always a whole number of minutes inside the bounds', () => { @@ -160,7 +168,7 @@ test('the result is always a whole number of minutes inside the bounds', () => { for (const input of cases) { const n = nextInterval({ ...input, now: NOW }); assert.ok(Number.isInteger(n), `whole minutes, got ${n} for ${JSON.stringify(input).slice(0, 60)}`); - assert.ok(n >= MIN_INTERVAL && n <= MAX_INTERVAL, `within bounds, got ${n}`); + assert.ok(n >= DATED_MIN_INTERVAL && n <= MAX_INTERVAL, `within bounds, got ${n}`); } }); diff --git a/packages/ingest/test/changes.test.js b/packages/ingest/test/changes.test.js index e656341..da914e4 100644 --- a/packages/ingest/test/changes.test.js +++ b/packages/ingest/test/changes.test.js @@ -125,10 +125,10 @@ test('an undated feed is scheduled on the rhythm we observed', () => { // than instants the publisher claimed: half the typical gap while the feed is // keeping to it. const weekly = intervalFromChanges(log(8, 7, 1), NOW); - assert.ok(days(weekly) > 3 && days(weekly) < 4, `weekly -> ${days(weekly).toFixed(1)}d`); + assert.equal(days(weekly), 1.75, `weekly -> ${days(weekly).toFixed(2)}d`); const daily = intervalFromChanges(log(8, 1, 0.2), NOW); - assert.ok(days(daily) > 0.4 && days(daily) < 0.6, `daily -> ${days(daily).toFixed(2)}d`); + assert.equal(days(daily), 0.25, `daily -> ${days(daily).toFixed(2)}d`); }); test('one observation is enough, which a single date never is', () => { @@ -138,7 +138,7 @@ test('one observation is enough, which a single date never is', () => { // since and seen nothing" -- because the looking is ours. So silence measured // from it is evidence, and a feed decays on it with no dead-feed classifier. assert.equal(intervalFromChanges(log(1, 0, 0), NOW), MIN_INTERVAL, 'just seen, so the floor'); - assert.ok(days(intervalFromChanges(log(1, 0, 40), NOW)) > 9, 'quiet 40 days -> read every 10'); + assert.equal(days(intervalFromChanges(log(1, 0, 40), NOW)), 7, 'quiet 40 days -> read weekly'); assert.equal(intervalFromChanges(log(1, 0, 2000), NOW), MAX_INTERVAL, 'and it stops at the ceiling'); }); diff --git a/packages/ingest/test/recrawl.test.js b/packages/ingest/test/recrawl.test.js index 8739581..b7c1852 100644 --- a/packages/ingest/test/recrawl.test.js +++ b/packages/ingest/test/recrawl.test.js @@ -114,7 +114,8 @@ test('a feed is scheduled on its own rhythm rather than on a fixed ladder', asyn await crawlFeed(db, feed, { resolve }); const first = Number((await q.feedBySlug(db, 'quiet')).fetch_interval_minutes); - assert.ok(first / 1440 > 3 && first / 1440 < 4, `weekly blog -> ${(first / 1440).toFixed(1)} days`); + // Six hours on release day up to 1.75 days just after a post, by where in its week it is. + assert.ok(first >= 360 && first <= 2520, `weekly blog -> ${(first / 1440).toFixed(2)} days`); // Crawling it again does not ratchet it anywhere: same document, same answer. await crawlFeed(db, await q.feedBySlug(db, 'quiet'), { resolve }); @@ -445,7 +446,7 @@ test('a feed that starts publishing again accelerates on its first new post', as const revived = await q.feedBySlug(db, 'quiet'); const back = Number(revived.fetch_interval_minutes) / 1440; - assert.ok(back > 3 && back < 4, `back to a weekly rhythm, not ${back.toFixed(0)} days`); + assert.equal(back, 1.75, `back to a weekly rhythm, not ${back.toFixed(0)} days`); }); test('a scraped source is never asked conditionally', async () => {