Refresh button + tiered crawl queue + archive silent feeds

apps/poller/package.json

@@ -8,7 +8,7 @@
     "test": "node --test test/*.test.js"
   },
   "dependencies": {
-    "@profullstack/scraper-daemon": "0.2.2",
+    "@profullstack/scraper-daemon": "0.3.0",
     "@rssamplifier/db": "workspace:*",
     "@rssamplifier/discover": "workspace:*",
     "@rssamplifier/feed": "workspace:*",

apps/poller/src/index.js

@@ -1022,6 +1022,11 @@ async function queueTick() {
       cards: backlogs.cardsPending,
       authors,
     });
+
+    // Due feeds per crawl tier and how late the oldest of each is: the line that
+    // says live feeds are being read on time while the quiet tiers drain.
+    const tiers = await q.dueByTier(db);
+    log('tiers', Object.fromEntries(tiers.map((t) => [t.tier, `${t.due} due, oldest ${t.oldest ?? '-'}`])));
   } catch (err) {
     // A missed sample is a gap in a chart, which the chart draws as a gap. It
     // is not a reason to make noise in a log that is read for crawl failures.

apps/web/src/app/RefreshFeed.jsx

@@ -0,0 +1,54 @@
+'use client';
+
+import { useRouter } from 'next/navigation';
+import { useState } from 'react';
+
+const SAID = {
+  recent: () => 'Already read in the last few minutes.',
+  read: (r) => (r.newItems > 0 ? `${r.newItems} new ${r.newItems === 1 ? 'post' : 'posts'}.` : 'Read just now: nothing new.'),
+  unchanged: () => 'Read just now: nothing new.',
+  queued: () => 'Queued: this source is read by the crawler, first in line.',
+  reading: () => 'Still reading; this page will catch up shortly.',
+  throttled: () => 'The publisher asked us to slow down; try again later.',
+  failed: (r) => `Could not read it: ${r.error ?? 'the publisher did not answer'}.`,
+  dead: (r) => r.error,
+};
+
+/**
+ * Read this feed now instead of waiting for its turn. POSTs to
+ * /api/feeds/<slug>/refresh, which runs the crawler's own read, then re-renders
+ * the page so new posts and the freshness line show up in place.
+ *
+ * @param {{ slug: string }} props
+ */
+export default function RefreshFeed({ slug }) {
+  const router = useRouter();
+  const [busy, setBusy] = useState(false);
+  const [said, setSaid] = useState('');
+
+  async function refresh() {
+    setBusy(true);
+    setSaid('Reading the feed…');
+    try {
+      const res = await fetch(`/api/feeds/${encodeURIComponent(slug)}/refresh`, { method: 'POST' });
+      const body = await res.json().catch(() => ({}));
+      setSaid((SAID[body.state] ?? SAID.failed)(body));
+      if (body.ok) router.refresh();
+    } catch {
+      setSaid('Could not reach the server.');
+    } finally {
+      setBusy(false);
+    }
+  }
+
+  return (
+    <p className="refresh-feed">
+      <button type="button" className="refresh-feed-button" onClick={refresh} disabled={busy}>
+        {busy ? 'Refreshing…' : 'Refresh feed'}
+      </button>
+      <span className="refresh-feed-said" aria-live="polite">
+        {said}
+      </span>
+    </p>
+  );
+}

apps/web/src/app/[slug]/page.jsx

@@ -23,6 +23,7 @@ import FollowControls from '../FollowControls.jsx';
 import ListFilter from '../ListFilter.jsx';
 import { FILTER_FROM } from '../../lib/listFilter.js';
 import Freshness from '../Freshness.jsx';
+import RefreshFeed from '../RefreshFeed.jsx';
 import PlayButton from '../PlayButton.jsx';
 import QueueAll from '../QueueAll.jsx';
 import QueueButton from '../QueueButton.jsx';
@@ -326,6 +327,7 @@ export default async function FeedPage({ params, base }) {
           lib/freshness.js. The newest post's date comes off the list this page
           has already loaded, so the signal costs no extra query. */}
       <Freshness feed={feed} newestPost={posts[0]?.published_at ?? null} />
+      <RefreshFeed slug={feed.slug} />
 
       {/* Follow and share, side by side, because they are the two things to do
           with a feed you have just found and only one of them needs an

apps/web/src/app/api/feeds/[slug]/refresh/route.js

@@ -0,0 +1,82 @@
+import { q } from '@rssamplifier/db';
+import { crawlFeed } from '@rssamplifier/ingest';
+import { isCollected } from '@rssamplifier/social';
+
+import { db } from '../../../../../lib/db.js';
+
+export const dynamic = 'force-dynamic';
+
+/** A feed read this recently is answered from what we have: a click is not a reason to ask the publisher twice. */
+const RECENT_MS = 5 * 60_000;
+
+/** How long a click waits on the publisher before it is told the read carries on without it. */
+const WAIT_MS = 25_000;
+
+/** Reads this worker has in flight, so a double click is one request to the publisher. */
+const inflight = new Map();
+
+const json = (body, status = 200) =>
+  new Response(JSON.stringify(body), {
+    status,
+    headers: {
+      'content-type': 'application/json; charset=utf-8',
+      'cache-control': 'no-store',
+      'access-control-allow-origin': '*',
+    },
+  });
+
+/**
+ * Read one feed now: the refresh button on a feed page (and anyone's agent).
+ *
+ * The crawler reads every feed on its own schedule, but a schedule is a guess
+ * and a reader looking at a stale page knows better. This runs the crawler's
+ * own crawlFeed in the request, so what lands is exactly what a scheduled read
+ * would have stored, and the cadence it writes is recomputed from the fresh
+ * document. Social sources are collected by the poller's runtime, not here, so
+ * those are moved to the front of their queue instead.
+ *
+ * @param {Request} _req
+ * @param {{ params: Promise<{ slug: string }> }} ctx
+ */
+export async function POST(_req, { params }) {
+  const { slug } = await params;
+  const client = db();
+  const feed = await q.feedBySlug(client, slug);
+  if (!feed) return json({ error: 'not-found' }, 404);
+  if (feed.status === 'dead') return json({ ok: false, state: 'dead', error: 'This feed has been retired after repeated failures.' }, 409);
+
+  const last = Date.parse(String(feed.last_fetched_at ?? ''));
+  if (Number.isFinite(last) && Date.now() - last < RECENT_MS) {
+    return json({ ok: true, state: 'recent', checkedAt: new Date(last).toISOString(), newItems: 0 });
+  }
+
+  if (isCollected(feed)) {
+    await q.requestRefresh(client, String(feed.id));
+    return json({ ok: true, state: 'queued' }, 202);
+  }
+
+  const id = String(feed.id);
+  let run = inflight.get(id);
+  if (!run) {
+    run = crawlFeed(client, feed)
+      .catch((err) => ({ ok: false, error: String(err?.message ?? err) }))
+      .finally(() => inflight.delete(id));
+    inflight.set(id, run);
+  }
+
+  const res = await Promise.race([run, new Promise((resolve) => setTimeout(() => resolve(null), WAIT_MS))]);
+  if (res === null) return json({ ok: true, state: 'reading' }, 202);
+  if (!res.ok) {
+    return json({ ok: false, state: res.throttled ? 'throttled' : 'failed', error: res.error ?? 'The publisher did not answer.' }, 502);
+  }
+
+  const after = await q.feedBySlug(client, slug);
+  return json({
+    ok: true,
+    state: res.notModified ? 'unchanged' : 'read',
+    newItems: Number(res.newItems ?? 0),
+    checkedAt: after?.last_success_at ?? null,
+    lastPublishedAt: after?.last_published_at ?? null,
+    nextCheckAt: after?.next_fetch_at ?? null,
+  });
+}

apps/web/src/app/api/feeds/[slug]/route.js

@@ -62,6 +62,7 @@ export async function GET(req, { params }) {
         freshness: fresh.state,
         lastPublishedAt: fresh.publishedAt,
         nextCheckAt: fresh.nextCheckAt,
+        archived: fresh.archived,
         page: `${siteUrl()}/${feed.slug}`,
         topics: topics.map((t) => ({
           slug: t.slug,

apps/web/src/app/globals.css

@@ -3932,6 +3932,36 @@ html:not([data-dock]) .playlist-hint {
   border-bottom: 1px dotted var(--line);
 }
 
+.refresh-feed {
+  display: flex;
+  flex-wrap: wrap;
+  align-items: center;
+  gap: 0.6rem;
+  margin: 0.5rem 0 0;
+  font-family: var(--mono);
+  font-size: 0.74rem;
+  color: var(--muted);
+}
+
+.refresh-feed-button {
+  font: inherit;
+  cursor: pointer;
+  padding: 0.3rem 0.7rem;
+  border: 1px solid var(--line);
+  border-radius: 4px;
+  background: transparent;
+  color: var(--fg);
+}
+
+.refresh-feed-button:hover:not(:disabled) {
+  border-color: var(--accent);
+}
+
+.refresh-feed-button:disabled {
+  cursor: progress;
+  opacity: 0.6;
+}
+
 .freshness-note {
   display: inline-block;
   margin-top: 0.35rem;

apps/web/src/lib/freshness.js

@@ -45,6 +45,12 @@ const OVERDUE_MULTIPLE = 2;
 /** Below this, a feed that has not published is called dormant rather than quiet. */
 const DORMANT_DAYS = 365;
 
+/** Past a week between reads, a feed is archived (see @profullstack/scraper-daemon/cadence). */
+const ARCHIVED_AFTER_INTERVAL = 7 * DAY;
+
+/** ...and only once it has been silent this long. */
+const ARCHIVED_AFTER_SILENCE = 180 * DAY;
+
 /**
  * A short, plain interval. "3 minutes", "4 hours", "2 days", "8 months".
  *
@@ -115,6 +121,10 @@ export function freshness(feed = {}, newestPost = null, now = Date.now()) {
     publishedAt: published === null ? null : new Date(published).toISOString(),
     publishedGap: published === null ? null : humanGap(now - published),
     nextCheckAt: next === null ? null : new Date(next).toISOString(),
+    // Archived: silent six months, so the crawler reads it once a quarter rather
+    // than with the live feeds. The house cadence marks that with an interval
+    // past a week (the longest a live feed ever waits), so the interval says it.
+    archived: interval > ARCHIVED_AFTER_INTERVAL && published !== null && now - published > ARCHIVED_AFTER_SILENCE,
   };
 
   // Never successfully read. Distinct from every other state: we are not
@@ -153,6 +163,14 @@ export function freshness(feed = {}, newestPost = null, now = Date.now()) {
   // Read on time, but the publisher has gone quiet. This is the one the reader
   // most needs and is least likely to guess: the data is current, and current
   // means "nothing since 2023".
+  if (base.archived) {
+    return {
+      ...base,
+      state: 'dormant',
+      note: `Archived: nothing published for ${humanGap(now - published)}. Kept for the record and checked once a quarter in case it returns.`,
+    };
+  }
+
   if (published !== null && now - published > DORMANT_DAYS * DAY) {
     return {
       ...base,

apps/web/test/freshness.test.js

@@ -146,3 +146,12 @@ test('every state has a label, so the badge can never render undefined', () => {
   for (const s of reached) assert.ok(states.includes(s), `${s} is a known state`);
   assert.equal(reached.size, states.length, 'all five states are reachable');
 });
+
+test('an archived feed reads dormant and says it is checked once a quarter', () => {
+  const archived = { ...live, last_success_at: iso(30 * DAY), fetch_interval_minutes: 90 * 24 * 60 };
+  const f = freshness(archived, iso(200 * DAY), NOW);
+  assert.equal(f.archived, true);
+  assert.equal(f.state, 'dormant');
+  assert.match(f.note, /once a quarter/);
+  assert.equal(freshness({ ...archived, fetch_interval_minutes: 60 }, iso(200 * DAY), NOW).archived, false);
+});

packages/db/migrations-pg/20261029010615_feeds_due_tier_idx.sql

@@ -0,0 +1,12 @@
+-- The crawl tier, as the dispatcher reads it: due feeds per tier, oldest due
+-- first (queries.js dueFeeds / TIER_SQL). The expression must stay exactly what
+-- tierSql('fetch_interval_minutes') from @profullstack/scraper-daemon/cadence
+-- returns, or the planner will not use this index.
+--
+-- Concurrently, because feeds is read by every page: migrate() runs each
+-- statement on its own with no transaction around the file, which is what
+-- create index concurrently needs. Built by hand on production first, so on
+-- boot this is a no-op.
+create index concurrently if not exists feeds_due_tier_idx
+  on feeds ((case when fetch_interval_minutes <= 60 then 0 when fetch_interval_minutes <= 1440 then 1 when fetch_interval_minutes < 10080 then 2 when fetch_interval_minutes <= 10080 then 3 else 4 end), next_fetch_at)
+  where status <> 'dead';

packages/db/package.json

@@ -24,6 +24,7 @@
     "bullmq": "^6.4.2",
     "ioredis": "^6.0.0",
     "pg": "^8.23.1",
-    "pg-copy-streams": "^7.0.0"
+    "pg-copy-streams": "^7.0.0",
+    "@profullstack/scraper-daemon": "0.3.0"
   }
 }