From 7d032a9285eaa417a4fa374e18bbe3f28b246c4c Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Fri, 14 Aug 2026 10:54:42 +0000 Subject: [PATCH] news: carry smallweb as one feed instead of a 33k-feed catalogue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `find` loads every published list on every search, and smallweb was a 5MB OPML of 33,000 outlines plus a 1.1MB plain-text fork of the same thing. Marking them search-only kept them out of a subscription but not out of a search, so `/rss` and `/news find` paid for parsing 33,000 feeds each time and fell over. Paging a catalogue that size is a feature nobody asked for. Kagi publishes those same blogs already merged: kagi.com/smallweb/feed is one Atom document, newest first, with titles and summaries. So the two catalogue entries are replaced by a `format: "feed"` list — one feed wearing a list's name, subscribed without fetching anything and tagged with the list so `rm smallweb` still takes it back out. Two guards come with it, because an aggregated feed has no natural size: · Feed reads truncate at 2MB rather than failing at 8MB. The firehose is 10MB, which the old cap refused outright, leaving the reader with nothing when the newest few hundred entries were what it wanted. · No feed may put more than MAX_LIMIT items into a merged listing. Without that, subscribing to an aggregator is the same as unsubscribing from everything else: 4,000 posts from today sort above every other feed's headlines. Co-Authored-By: Claude Opus 5 (1M context) --- src/news-sources.mjs | 38 +++++++------- src/news.mjs | 120 +++++++++++++++++++++++++++++++++++++------ test/news.test.mjs | 91 ++++++++++++++++++++++++++++---- 3 files changed, 203 insertions(+), 46 deletions(-) diff --git a/src/news-sources.mjs b/src/news-sources.mjs index 3eb6d774..a9dc17a7 100644 --- a/src/news-sources.mjs +++ b/src/news-sources.mjs @@ -272,16 +272,24 @@ export const DEFAULT_FEEDS = [ * one. They are somebody else's lists, and that is the point: the feeds inside * them stay current without moshcode shipping a release. * - * Two shapes, because the lists worth reading come in two: + * Three shapes, because the lists worth reading come in three: * * `format: "opml"` — an OPML document, parsed by parseOpml. - * `format: "text"` — one feed URL per line, `#` comments ignored. Kagi's - * smallweb list is published this way and there is no OPML of it. + * `format: "text"` — one feed URL per line, `#` comments ignored. + * `format: "feed"` — the URL is not a catalogue at all but a single feed that + * already aggregates one. Nothing is fetched to list it; `add` subscribes + * to the one URL. This is how smallweb is carried — see below. * - * `searchOnly` marks a list too large to subscribe to wholesale. smallweb is - * 32,000+ feeds: importing it would write a 32,000-entry news.opml and then try - * to fetch every one of them on the next `/news`. It is a catalogue to search, - * not a subscription — `/rss search ` is how you get feeds out of it. + * Kagi's Small Web used to be here as its catalogue: a 5MB OPML of 33,000 + * personal blogs, plus a 1.1MB plain-text fork of the same thing. Both were + * marked search-only, because subscribing to 33,000 feeds is not a thing a + * reader can do — but `find` reads every list on every search, so *searching* + * loaded and parsed 33,000 outlines each time, which is what took `/rss` and + * `/news find` down. Lazily paging a catalogue that size is a feature nobody + * asked for, so the catalogue is gone and the firehose stands in for it: + * kagi.com/smallweb/feed is those same blogs already merged into one Atom + * document, newest first, with titles and summaries. One feed, one fetch, and + * `/news --feed smallweb` reads it on its own. */ export const FEED_LISTS = [ { @@ -310,17 +318,11 @@ export const FEED_LISTS = [ }, { name: "smallweb", - description: "Kagi Small Web — 33k personal blogs, with titles", - url: "https://kagi.com/smallweb/opml", - format: "opml", - searchOnly: true, - }, - { - name: "smallweb-txt", - description: "Kagi Small Web, profullstack's fork — the plain-text list", - url: "https://raw.githubusercontent.com/ralyodio/smallweb/refs/heads/main/smallweb.txt", - format: "text", - searchOnly: true, + description: "Kagi Small Web — 33k personal blogs, merged into one feed", + url: "https://kagi.com/smallweb/feed", + format: "feed", + title: "Kagi Small Web", + site: "https://kagi.com/smallweb", }, ]; diff --git a/src/news.mjs b/src/news.mjs index 278efc1c..7bb36c8a 100644 --- a/src/news.mjs +++ b/src/news.mjs @@ -99,6 +99,31 @@ const FILE_MODE = 0o600; /** Enough for a large feed, small enough that one bad URL cannot eat the pit. */ const MAX_BYTES = 8 * 1024 * 1024; +/** + * Where a *feed* read stops, and — unlike MAX_BYTES — stopping is not failing. + * + * An aggregated feed has no natural size: smallweb's firehose is 10MB of + * today's posts, which the 8MB cap would refuse outright, leaving the reader + * with nothing rather than with the newest few hundred entries. Feeds are + * newest-first, so the first two megabytes are the part worth having; the rest + * of the transfer is cancelled mid-stream and what arrived is parsed. parseFeed + * matches whole ``/`` blocks, so a cut tail simply is not an item. + * + * Ordinary feeds never reach this — the largest in the defaults is under 1MB. + */ +const FEED_MAX_BYTES = 2 * 1024 * 1024; + +/** + * The most headlines one feed may put into a merged listing. + * + * Without it, subscribing to an aggregator is the same as unsubscribing from + * everything else: the merge sorts by date, and 4,000 small-web posts from + * today's firehose sit above every headline the other feeds have. Set at + * MAX_LIMIT so no listing can be starved by the cap — `--feed smallweb --limit + * 200` still fills, because a single feed can supply every row. + */ +const PER_FEED_ITEMS = 200; + /** How many feeds are in flight at once. Politeness, not throughput. */ const CONCURRENCY = 6; @@ -413,11 +438,11 @@ export function findFeed(feeds, needle) { /** * Parse a newline-delimited feed list: one URL per line, `#` comments ignored. * - * The other shape a published list comes in. Kagi's smallweb list is a plain - * text file of 32,000 URLs and there is no OPML of it, so a reader that only - * speaks OPML cannot read the largest list there is. No titles are invented - * beyond the hostname — naming 32,000 feeds properly would mean fetching - * 32,000 feeds, and the host is what the URL already tells us for free. + * The other shape a published list comes in: plenty of them are a text file of + * URLs and nothing else, so a reader that only speaks OPML cannot read them. + * No titles are invented beyond the hostname — naming a list's feeds properly + * would mean fetching every one of them, and the host is what the URL already + * tells us for free. */ export function parseFeedList(text) { const feeds = []; @@ -440,7 +465,7 @@ export function parseListDocument(body, format = "") { return looksLikeOpml(body) ? parseOpml(body) : parseFeedList(body); } -/** Where a fetched list is cached, so searching does not refetch 32k feeds. */ +/** Where a fetched list is cached, so searching does not refetch every list. */ export function listCacheFile(name, env = process.env) { return path.join(path.dirname(opmlFile(env)), "lists", `${slugify(name) || "list"}.json`); } @@ -462,6 +487,24 @@ const LIST_CACHE_MS = 24 * 60 * 60 * 1000; export async function loadListFeeds(list, { fetchImpl, env = process.env, timeoutMs = DEFAULT_TIMEOUT_MS, now = Date.now(), refresh = false, } = {}) { + // A `format: "feed"` list is a single feed that has already done the + // aggregating — smallweb's firehose. There is no catalogue to fetch or cache: + // the entry itself is the one feed, so a search costs nothing and can never + // be the thing that makes `find` slow. + if (list.format === "feed") { + return { + ok: true, + cached: false, + feeds: [{ + name: list.name, + title: list.title || list.name, + url: list.url, + site: list.site || "", + category: list.name, + }], + }; + } + const file = listCacheFile(list.name, env); const cached = () => { try { @@ -512,7 +555,7 @@ export function parseKeywords(raw) { * * Ranked rather than filtered, because filtering cannot win both halves of this: * - * · Plain substring matching searches 32,000 feeds for `rust` and returns + * · Plain substring matching searches every list for `rust` and returns * Trust Machines, Trustnodes, frustrat.com and popthruster.com. * · Requiring the keyword to start a word fixes that and then finds nothing * at all for `homelab`, because the list's only metadata is the hostname @@ -689,8 +732,15 @@ export function tidyTitle(title) { * * Size-capped while streaming rather than after: a feed that turns out to be a * disk image should cost a few megabytes of transfer, not all of it. + * + * What happens at the cap is the caller's to choose. A document read whole — + * an OPML catalogue — fails, because half a catalogue is a wrong answer rather + * than a partial one. A feed read for its newest entries truncates instead: + * see FEED_MAX_BYTES. */ -export async function fetchDocument(url, { fetchImpl, timeoutMs = DEFAULT_TIMEOUT_MS } = {}) { +export async function fetchDocument(url, { + fetchImpl, timeoutMs = DEFAULT_TIMEOUT_MS, maxBytes = MAX_BYTES, truncate = false, +} = {}) { const impl = fetchImpl || globalThis.fetch; if (typeof impl !== "function") return { ok: false, error: "no fetch available in this runtime" }; const safe = safeUrl(url); @@ -713,7 +763,10 @@ export async function fetchDocument(url, { fetchImpl, timeoutMs = DEFAULT_TIMEOU // text() for any fetch implementation (tests included) that has no body. if (!res.body || typeof res.body.getReader !== "function") { const body = await res.text(); - if (body.length > MAX_BYTES) return { ok: false, error: `feed is larger than ${MAX_BYTES} bytes` }; + if (body.length > maxBytes) { + if (!truncate) return { ok: false, error: `feed is larger than ${maxBytes} bytes` }; + return { ok: true, body: body.slice(0, maxBytes), truncated: true }; + } return { ok: true, body }; } const reader = res.body.getReader(); @@ -724,9 +777,11 @@ export async function fetchDocument(url, { fetchImpl, timeoutMs = DEFAULT_TIMEOU const { done, value } = await reader.read(); if (done) break; bytes += value.byteLength; - if (bytes > MAX_BYTES) { + if (bytes > maxBytes) { try { await reader.cancel(); } catch { /* already gone */ } - return { ok: false, error: `feed is larger than ${MAX_BYTES} bytes` }; + if (!truncate) return { ok: false, error: `feed is larger than ${maxBytes} bytes` }; + body += decoder.decode(value, { stream: true }); + return { ok: true, body: body + decoder.decode(), truncated: true }; } body += decoder.decode(value, { stream: true }); } @@ -772,7 +827,7 @@ async function mapLimit(items, limit, worker) { */ export async function collectNews(feeds, { fetchImpl, timeoutMs = DEFAULT_TIMEOUT_MS } = {}) { const results = await mapLimit(feeds, CONCURRENCY, async (feed) => { - const res = await fetchDocument(feed.url, { fetchImpl, timeoutMs }); + const res = await fetchDocument(feed.url, { fetchImpl, timeoutMs, maxBytes: FEED_MAX_BYTES, truncate: true }); if (!res.ok) return { feed, error: res.error }; let parsed; try { parsed = parseFeed(res.body, { url: feed.url }); } @@ -787,7 +842,8 @@ export async function collectNews(feeds, { fetchImpl, timeoutMs = DEFAULT_TIMEOU let skipped = 0; for (const result of results) { if (result.error) { failures.push({ name: result.feed.name, url: result.feed.url, error: result.error }); continue; } - for (const item of result.parsed.items) { + // Newest first inside a feed, so the cap keeps the entries worth keeping. + for (const item of result.parsed.items.slice(0, PER_FEED_ITEMS)) { // Aggregator feeds wrap the publisher's URL in one of their own. Unwrap // before deduping, so the same story arriving via Google News and via the // publisher's own feed is recognised as one story rather than two. @@ -1398,9 +1454,13 @@ const FIND_LIMIT = 25; /** * `news find ` — search every published list for feeds to subscribe to. * - * This is the way into the big lists. smallweb alone is 32,000 feeds, which is - * a fine thing to search and an impossible thing to subscribe to, so the - * catalogue is read here and only the matches become candidates for `add`. + * This is the way into the lists: they hold a few thousand feeds between them, + * which is a fine thing to search and a poor thing to read end to end, so the + * catalogues are read here and only the matches become candidates for `add`. + * + * Every list is loaded on every search, which is why none of them may be + * enormous — a 33,000-feed catalogue in here made this the slowest thing in the + * pit and took `/rss` down with it. */ async function findCommand(request, { out, fail, fetchImpl, env }) { const hits = []; @@ -1418,7 +1478,7 @@ async function findCommand(request, { out, fail, fetchImpl, env }) { } // Ranked across every list, not within each one. Sorting per list and then - // concatenating puts all of web3's weak matches above smallweb's exact ones + // concatenating puts all of web3's weak matches above journalists' exact ones // purely because web3 is fetched first. hits.sort((a, b) => scoreFeed(b, request.keywords) - scoreFeed(a, request.keywords)); @@ -1472,6 +1532,32 @@ async function addCommand(request, { out, fail, fetchImpl, env }) { fail(` ${ash("too large to subscribe to wholesale — search it instead:")} ${bone(`/rss search `)}`); return 1; } + // A `format: "feed"` list is one feed wearing a list's name. Subscribed + // without fetching it first: smallweb's firehose is ten megabytes of + // headlines and not one of them is needed to decide the feed exists. Tagged + // with the list name like any other list, so `rm smallweb` still takes it + // back out. + if (bundle?.format === "feed") { + const { feeds, added, existed } = withFeed(loadFeeds(env), { + name: bundle.name, + title: bundle.title || bundle.name, + url: bundle.url, + site: bundle.site || "", + category: bundle.name, + }); + if (existed) { out(`${ash("· ")}already subscribed to ${bone(added.name)} ${ash(added.url)}`); return 0; } + try { saveFeeds(feeds, env); } + catch (e) { fail(danger(`✗ can't write ${opmlFile(env)}: ${e.message}`)); return 1; } + // Named from the OPML on the way back in, like every other feed, so the + // name printed here is the one `--feed` will answer to rather than the one + // that went in. + const saved = findFeed(loadFeeds(env), bundle.url) ?? added; + if (request.json) { out(JSON.stringify({ added: saved, skipped: 0 }, null, 2)); return 0; } + out(`${acid("✓ ")}subscribed to ${bone(saved.name)} ${ash(`— ${bundle.description}`)}`); + out(` ${ash("read it on its own with")} ${bone(`/news --feed ${saved.name}`)}`); + return 0; + } + const target = bundle ? bundle.url : request.target; if (bundle) out(`${ash("· ")}fetching ${bone(bundle.name)} ${ash(`— ${bundle.description}`)}`); diff --git a/test/news.test.mjs b/test/news.test.mjs index a24fe0e9..c29004d4 100644 --- a/test/news.test.mjs +++ b/test/news.test.mjs @@ -428,6 +428,50 @@ test("one dead publisher is reported, not fatal", async () => { assert.match(failures[0].error, /404/); }); +/** A feed of `n` dated entries, newest first, each padded to `pad` bytes. */ +function firehose(n, pad = 0) { + const entries = Array.from({ length: n }, (_, i) => ` + Post ${i + 1} + https://small.example/${i + 1} + Tue, 11 Aug 2026 12:00:00 GMT + ${"x".repeat(pad)} + `); + return `Firehose${entries.join("")}`; +} + +test("an aggregated feed is truncated to its newest entries, not refused", async () => { + // Kagi's small-web firehose is 10MB of today's posts — over the cap that a + // catalogue fetch fails on. Failing here would leave the reader with nothing + // when the first two megabytes are exactly what it wanted. + const huge = firehose(4000, 600); + assert.ok(huge.length > 2 * 1024 * 1024, "fixture must exceed the feed cap"); + const { items, failures } = await collectNews( + [{ name: "smallweb", url: "https://kagi.com/smallweb/feed" }], + { fetchImpl: fakeFetch({ "https://kagi.com/smallweb/feed": huge }) }, + ); + assert.deepEqual(failures, []); + assert.ok(items.length > 0, "a truncated feed still has headlines"); + assert.equal(items[0].title, "Post 1"); +}); + +test("one feed cannot take over a merged listing", async () => { + // Without a per-feed cap, subscribing to an aggregator is the same as + // unsubscribing from everything else: 4,000 posts from today sort above + // every other feed's headlines. + const { items } = await collectNews([ + { name: "smallweb", url: "https://kagi.com/smallweb/feed" }, + { name: "wire", url: "https://a.example/rss" }, + ], { + fetchImpl: fakeFetch({ + "https://kagi.com/smallweb/feed": firehose(500), + "https://a.example/rss": RSS, + }), + }); + assert.equal(items.filter((i) => i.feed === "smallweb").length, 200); + assert.deepEqual(items.filter((i) => i.feed === "wire").map((i) => i.title), + ["First story", "Second story"]); +}); + test("undated entries sort last rather than claiming to be the newest", async () => { const undated = `U No datehttps://u.example/1`; @@ -925,8 +969,11 @@ test("find searches the published lists and numbers what it found", async () => ...io, env, fetchImpl: fakeFetch({ "https://profullstack.com/feeds.opml": OPML, - "https://raw.githubusercontent.com/ralyodio/smallweb/refs/heads/main/smallweb.txt": - "https://example.com/blog/rss.xml\nhttps://unrelated.test/rss\n", + "https://raw.githubusercontent.com/CoinFabrik/resources/master/decentralized-and-blockchain-feeds.opml": + ` + + + `, }), }); assert.equal(code, 0); @@ -937,8 +984,8 @@ test("find searches the published lists and numbers what it found", async () => test("add subscribes to what find numbered, and only what it showed", async () => { const { env } = sandbox(); const fetchImpl = fakeFetch({ - "https://raw.githubusercontent.com/ralyodio/smallweb/refs/heads/main/smallweb.txt": - "https://example.com/feed.xml\n", + "https://raw.githubusercontent.com/CoinFabrik/resources/master/decentralized-and-blockchain-feeds.opml": + ``, "https://example.com/feed.xml": RSS, }); await newsCommand(["find", "example.com"], { ...sink(), env, fetchImpl }); @@ -953,14 +1000,36 @@ test("add subscribes to what find numbered, and only what it showed", async assert.match(missing.errorText(), /no result 9/); }); -test("a search-only list refuses to be subscribed to wholesale", async () => { +test("smallweb subscribes as one feed, without downloading it first", async () => { + // The catalogue it used to be — 33,000 outlines — is what took `/rss` down. + // What replaced it is a single feed, and `add` must not fetch ten megabytes + // of headlines to find that out. + const { env } = sandbox(); const io = sink(); const code = await newsCommand(["add", "smallweb"], { - ...io, env: sandbox().env, - fetchImpl: () => { throw new Error("must not fetch a list it is going to refuse"); }, + ...io, env, + fetchImpl: () => { throw new Error("must not fetch a feed it is only subscribing to"); }, }); - assert.equal(code, 1); - assert.match(io.errorText(), /too large to subscribe to wholesale/); + assert.equal(code, 0); + const feeds = loadFeeds(env); + assert.deepEqual(feeds.map((f) => f.url), ["https://kagi.com/smallweb/feed"]); + // Tagged with the list it came from, so `rm smallweb` takes it back out. + assert.deepEqual(feeds.map(listOf), ["smallweb"]); + + const removed = sink(); + assert.equal(await newsCommand(["rm", "smallweb"], { ...removed, env }), 0); + assert.deepEqual(loadFeeds(env), []); +}); + +test("a one-feed list is searchable without a catalogue fetch", async () => { + const list = resolveBundle("smallweb"); + assert.equal(list.format, "feed"); + const loaded = await loadListFeeds(list, { + fetchImpl: () => { throw new Error("must not fetch"); }, + env: sandbox().env, + }); + assert.equal(loaded.ok, true); + assert.deepEqual(loaded.feeds.map((f) => f.url), ["https://kagi.com/smallweb/feed"]); }); test("a list is added and removed as one unit", async () => { @@ -1019,7 +1088,7 @@ test("a stale cache is served when the list cannot be fetched", async () => { }); test("find ranks across every list, not within each one", async () => { - // web3 is fetched before smallweb. Sorting per list and concatenating put + // web3 is fetched before blockchain. Sorting per list and concatenating put // "Trust Machines" above "rust.example" purely because of fetch order. const { env } = sandbox(); const io = sink(); @@ -1028,7 +1097,7 @@ test("find ranks across every list, not within each one", async () => { fetchImpl: fakeFetch({ "https://raw.githubusercontent.com/chainfeeds/RSSAggregatorforWeb3/main/RAW.opml": ``, - "https://kagi.com/smallweb/opml": + "https://raw.githubusercontent.com/CoinFabrik/resources/master/decentralized-and-blockchain-feeds.opml": ``, }), });