diff --git a/README.md b/README.md index 393c2acf..a86e4b79 100644 --- a/README.md +++ b/README.md @@ -236,6 +236,7 @@ All send `access-control-allow-origin: *` and need no key. | `/api/discoveries/{id}` | Progress of one keyword run | | `/api/crawlstats` | Crawler health, including the two discovery queues | | `/api/topics/{keyword}` | The feeds on a topic, its category breakdown, `?group=` to narrow | +| `/api/topics/{a},{b},{c}` | Several topics (up to 10) as one: the union, each feed once, tagged-under-any first then strength summed across them; adds `topics` and `unknown`. Works on every topic surface: `/topics/a,b`, `/topics/a,b/blogs`, `/topics/a,b.rss`, the player, and the MCP `get_topic`/`topic_posts` | | `/topics/{keyword}/{group}.rss` | One category of a topic, as a feed — also `.atom`, `.json`, `.m3u`, `.pls` | | `/mcp` | MCP endpoint — and the documentation page, in a browser | | `/api/authors` | The people behind the feeds; `?feed={url}` finds the people behind one feed | diff --git a/apps/web/package.json b/apps/web/package.json index 9547cfb1..10356ba1 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "@rssamplifier/web", - "version": "0.2.4", + "version": "0.3.0", "private": true, "type": "module", "scripts": { diff --git a/apps/web/src/app/api/queue/route.js b/apps/web/src/app/api/queue/route.js index b920eb1a..5aa81931 100644 --- a/apps/web/src/app/api/queue/route.js +++ b/apps/web/src/app/api/queue/route.js @@ -93,7 +93,10 @@ export async function POST(req) { const group = body.group ? topicGroup(body.group) : null; if (body.group && !group) return wantsHtml ? redirect(back) : json({ error: 'not-found' }, 404); - const rows = await q.mediaForTopic(client, String(body.topic ?? ''), { + // A combined view (business,saas) posts its slugs joined by commas, the + // same form its address uses; a single topic is a one-element list, which + // is the single-topic query unchanged. + const rows = await q.mediaForTopic(client, String(body.topic ?? '').split(','), { limit: PLAYLIST_LIMIT, kinds: group?.kinds ?? null, }); diff --git a/apps/web/src/app/api/topics/[keyword]/route.js b/apps/web/src/app/api/topics/[keyword]/route.js index 47630df3..a0b5d9da 100644 --- a/apps/web/src/app/api/topics/[keyword]/route.js +++ b/apps/web/src/app/api/topics/[keyword]/route.js @@ -1,7 +1,8 @@ import { q } from '@rssamplifier/db'; import { db, siteUrl } from '../../../../lib/db.js'; -import { groupsWithFeeds, slugFromUrl, topicGroup } from '../../../../lib/topicGroups.js'; +import { groupsWithFeeds, topicGroup } from '../../../../lib/topicGroups.js'; +import { resolveTopics, topicQueryArg, topicSegment } from '../../../../lib/topicSet.js'; export const dynamic = 'force-dynamic'; @@ -19,12 +20,19 @@ export const dynamic = 'force-dynamic'; * a second request, and a caller that guessed wrong can see what it should have * said. * + * Several topics at once — `/api/topics/business,saas,crypto` — answer in the + * same shape for their union (see lib/topicSet.js): every feed filed under any + * of them, each once, `total` counting distinct feeds. Three fields are added + * for a list: `topics` (each known topic with its own count and address), + * `unknown` (what was asked for and does not exist) and `strength` per feed is + * the sum across the topics. A list with nothing known is a 404; one longer + * than ten topics a 400. + * * @param {Request} req * @param {{ params: Promise<{ keyword: string }> }} ctx */ export async function GET(req, { params }) { const { keyword } = await params; - const slug = slugFromUrl(keyword); const url = new URL(req.url); const limit = Math.min(Math.max(Number(url.searchParams.get('limit') ?? 100) || 100, 1), 500); @@ -32,9 +40,28 @@ export async function GET(req, { params }) { const group = topicGroup(url.searchParams.get('group')); const client = db(); - const topic = await q.topicBySlug(client, slug); + const found = await resolveTopics(client, keyword); + + if (found.error === 'too-many') { + return new Response( + JSON.stringify( + { error: 'too-many-topics', max: found.max, hint: `Combine at most ${found.max} topics.` }, + null, + 2, + ), + { + status: 400, + headers: { + 'content-type': 'application/json; charset=utf-8', + 'access-control-allow-origin': '*', + }, + }, + ); + } + const topic = found.topic; if (!topic) { + const slug = found.unknown.join(','); return new Response(JSON.stringify({ error: 'not-found', slug }, null, 2), { status: 404, headers: { @@ -45,11 +72,11 @@ export async function GET(req, { params }) { } const [rows, counts] = await Promise.all([ - q.feedsForTopic(client, slug, { limit, offset, kinds: group?.kinds ?? null }), - q.topicKindCounts(client, slug), + q.feedsForTopic(client, topicQueryArg(topic), { limit, offset, kinds: group?.kinds ?? null }), + q.topicKindCounts(client, topicQueryArg(topic)), ]); - const base = `${siteUrl()}/topics/${encodeURIComponent(topic.slug)}`; + const base = `${siteUrl()}/topics/${topicSegment(topic.slug)}`; return new Response( JSON.stringify( @@ -63,6 +90,20 @@ export async function GET(req, { params }) { ? group.kinds.reduce((sum, kind) => sum + (counts[kind] ?? 0), 0) : topic.feedCount, topicTotal: topic.feedCount, + // Only on a combined view, so a single topic's answer is byte for byte + // what it was. + ...(topic.multi + ? { + topics: (topic.topics ?? []).map((t) => ({ + slug: t.slug, + keyword: t.keyword, + feedCount: t.feedCount, + page: `${siteUrl()}/topics/${encodeURIComponent(t.slug)}`, + api: `${siteUrl()}/api/topics/${encodeURIComponent(t.slug)}`, + })), + unknown: topic.unknown ?? [], + } + : {}), group: group?.segment ?? null, groups: groupsWithFeeds(counts).map(({ group: entry, count }) => ({ group: entry.segment, @@ -92,6 +133,8 @@ export async function GET(req, { params }) { // a phrase counted across its writing. source: f.source, strength: Number(f.count ?? 0), + // Which of the listed topics this feed is filed under. + ...(topic.multi ? { topics: String(f.topics ?? '').split(',').filter(Boolean) } : {}), page: `${siteUrl()}/${f.slug}`, })), }, diff --git a/apps/web/src/app/topics/[keyword]/TopicListing.jsx b/apps/web/src/app/topics/[keyword]/TopicListing.jsx index fcd0984e..a2d605bb 100644 --- a/apps/web/src/app/topics/[keyword]/TopicListing.jsx +++ b/apps/web/src/app/topics/[keyword]/TopicListing.jsx @@ -14,6 +14,7 @@ import Share from '../../Share.jsx'; import { Avatar } from '../../Thumb.jsx'; import { feedImage } from '../../../lib/thumbs.js'; import { jsonLdScript } from '../../../lib/jsonld.js'; +import { topicQueryArg, topicSegment } from '../../../lib/topicSet.js'; /** Feeds per page. Matches the category pages. */ export const PAGE_SIZE = 60; @@ -71,8 +72,13 @@ function formatTitle(ext, what) { * moment one of them grew a feature, and the sub-groups exist precisely because * there are now eight of them per topic. * + * A combined view (/topics/business,saas,crypto) is the same page over the + * union of its topics — see lib/topicSet.js. What it adds is a link to each + * topic it combines and a note naming any that do not exist; what it leaves + * out is everything keyed on one topic's slug (following it, its webring). + * * @param {{ - * topic: { slug: string, keyword: string, feedCount: number }, + * topic: import('../../../lib/topicSet.js').ResolvedTopic, * counts: Record, * group?: import('../../../lib/topicGroups.js').TOPIC_GROUPS[number]|null, * page?: number, @@ -80,7 +86,8 @@ function formatTitle(ext, what) { */ export default async function TopicListing({ topic, counts, group = null, page = 1 }) { const client = db(); - const slug = encodeURIComponent(topic.slug); + const slug = topicSegment(topic.slug); + const multi = Boolean(topic.multi); const base = group ? `/topics/${slug}/${group.segment}` : `/topics/${slug}`; const total = group @@ -88,7 +95,7 @@ export default async function TopicListing({ topic, counts, group = null, page = : topic.feedCount; const [rows, user] = await Promise.all([ - q.feedsForTopic(client, topic.slug, { + q.feedsForTopic(client, topicQueryArg(topic), { limit: PAGE_SIZE, offset: (page - 1) * PAGE_SIZE, kinds: group?.kinds ?? null, @@ -99,7 +106,7 @@ export default async function TopicListing({ topic, counts, group = null, page = // Only asked once we know there is someone to ask about. A signed-out visitor // still gets the button — the endpoint sends them to sign in and back here — // so this decides what it says rather than whether it appears. - const follow = user + const follow = user && !multi ? await alerts.topicFollowState(client, String(user.id), topic.slug, group?.segment ?? '') : { following: false, alerts: false }; @@ -154,15 +161,40 @@ export default async function TopicListing({ topic, counts, group = null, page =

{group ? ( <> - Topic · {topic.keyword} + {multi ? 'Topics' : 'Topic'} ·{' '} + {topic.keyword} ) : ( - Topic + {multi ? 'Topics' : 'Topic'} )}

{heading}

{lede}

+ {/* What this combined view is made of, each topic on its own page, and + what was asked for that does not exist — said rather than silently + dropped, so a typo in one of five topics is visible. */} + {multi && ( +

+ Every feed filed under any of:{' '} + {(topic.topics ?? []).map((t, i) => ( + + {i > 0 && ' · '} + + {t.keyword} + {' '} + ({t.feedCount}) + + ))} + {(topic.unknown ?? []).length > 0 && ( + <> + {' '} + — no topic called {(topic.unknown ?? []).map((u) => `"${u}"`).join(', ')}. + + )} +

+ )} + {/* The topic's other categories. Shown on the sub-group pages too, so moving between them never costs a trip back through the topic — which is the whole reason for cutting the topic up. Only the groups that @@ -230,17 +262,19 @@ export default async function TopicListing({ topic, counts, group = null, page = form underneath, so it works with JavaScript off; with JavaScript it flips in place rather than reloading a page of sixty feeds to change two words on one button. */} - + {!multi && ( + + )} - Webring - - - View webring - + {!multi && ( + <> + + Webring + + + View webring + + + )} @@ -283,6 +321,7 @@ export default async function TopicListing({ topic, counts, group = null, page = {f.item_count} {f.category === 'podcast' ? 'episodes' : 'posts'} + {multi && f.topics && {String(f.topics).split(',').join(', ')}} ); diff --git a/apps/web/src/app/topics/[keyword]/TopicPlayer.jsx b/apps/web/src/app/topics/[keyword]/TopicPlayer.jsx index aeb9e615..629d9bbd 100644 --- a/apps/web/src/app/topics/[keyword]/TopicPlayer.jsx +++ b/apps/web/src/app/topics/[keyword]/TopicPlayer.jsx @@ -4,6 +4,7 @@ import { db } from '../../../lib/db.js'; import { currentUser } from '../../../lib/auth.js'; import { PLAYLIST_LIMIT } from '../../../lib/topicFeed.js'; import { playlistEntry, playerPath, queueRuntime, rawPlaylistPath } from '../../../lib/player.js'; +import { topicQueryArg, topicSegment } from '../../../lib/topicSet.js'; import PlaylistPlayer from '../../PlaylistPlayer.jsx'; import QueueAll from '../../QueueAll.jsx'; @@ -16,16 +17,16 @@ import QueueAll from '../../QueueAll.jsx'; * they differ only in the filter, and two copies would drift. * * @param {{ - * topic: { slug: string, keyword: string, feedCount: number }, + * topic: import('../../../lib/topicSet.js').ResolvedTopic, * group?: { segment: string, kinds: string[], heading: string, item: string }|null, * }} props */ export default async function TopicPlayer({ topic, group = null }) { const client = db(); - const slug = encodeURIComponent(topic.slug); + const slug = topicSegment(topic.slug); const [rows, user] = await Promise.all([ - q.mediaForTopic(client, topic.slug, { + q.mediaForTopic(client, topicQueryArg(topic), { limit: PLAYLIST_LIMIT, kinds: group?.kinds ?? null, }), diff --git a/apps/web/src/app/topics/[keyword]/[group]/page.jsx b/apps/web/src/app/topics/[keyword]/[group]/page.jsx index 2f6a4f0b..5ba982d8 100644 --- a/apps/web/src/app/topics/[keyword]/[group]/page.jsx +++ b/apps/web/src/app/topics/[keyword]/[group]/page.jsx @@ -1,9 +1,9 @@ import { notFound } from 'next/navigation'; -import { q } from '@rssamplifier/db'; import { db, siteUrl } from '../../../../lib/db.js'; import { feedAlternates } from '../../../../lib/subscribe.js'; -import { slugFromUrl, topicGroup } from '../../../../lib/topicGroups.js'; +import { topicGroup } from '../../../../lib/topicGroups.js'; +import { resolveTopicsWithCounts, topicSegment } from '../../../../lib/topicSet.js'; import { pageNumber } from '../../../CategoryIndex.jsx'; import TopicListing from '../TopicListing.jsx'; @@ -19,7 +19,7 @@ export const dynamic = 'force-dynamic'; * * @param {{ params: Promise<{ keyword: string, group: string }> }} props * @returns {Promise<{ - * topic: { slug: string, keyword: string, feedCount: number }, + * topic: import('../../../../lib/topicSet.js').ResolvedTopic, * counts: Record, * group: NonNullable>, * total: number, @@ -31,12 +31,7 @@ async function resolve({ params }) { if (!group) return null; const client = db(); - const slug = slugFromUrl(keyword); - - const [topic, counts] = await Promise.all([ - q.topicBySlug(client, slug), - q.topicKindCounts(client, slug), - ]); + const { topic, counts } = await resolveTopicsWithCounts(client, keyword); if (!topic) return null; const total = group.kinds.reduce((sum, kind) => sum + (counts[kind] ?? 0), 0); @@ -58,12 +53,14 @@ export async function generateMetadata(props) { if (!found) return { title: 'Not found' }; const { topic, group, total } = found; - const page = `${siteUrl()}/topics/${encodeURIComponent(topic.slug)}/${group.segment}`; + const page = `${siteUrl()}/topics/${topicSegment(topic.slug)}/${group.segment}`; const heading = `${topic.keyword}: ${group.heading.toLowerCase()}`; return { title: heading, description: `${total} ${group.noun} in the RSS Amplifier directory cover ${topic.keyword}.`, + // Out of the index for a combined view; see the whole-topic page. + ...(topic.multi ? { robots: { index: false, follow: true } } : {}), alternates: { canonical: page, types: feedAlternates(page, heading), diff --git a/apps/web/src/app/topics/[keyword]/[group]/play/page.jsx b/apps/web/src/app/topics/[keyword]/[group]/play/page.jsx index de5246bb..e644e5e8 100644 --- a/apps/web/src/app/topics/[keyword]/[group]/play/page.jsx +++ b/apps/web/src/app/topics/[keyword]/[group]/play/page.jsx @@ -1,8 +1,8 @@ import { notFound } from 'next/navigation'; -import { q } from '@rssamplifier/db'; import { db, siteUrl } from '../../../../../lib/db.js'; -import { slugFromUrl, topicGroup } from '../../../../../lib/topicGroups.js'; +import { topicGroup } from '../../../../../lib/topicGroups.js'; +import { resolveTopicsWithCounts, topicSegment } from '../../../../../lib/topicSet.js'; import TopicPlayer from '../../TopicPlayer.jsx'; export const dynamic = 'force-dynamic'; @@ -24,7 +24,7 @@ export const dynamic = 'force-dynamic'; * * @param {{ params: Promise<{ keyword: string, group: string }> }} props * @returns {Promise<{ - * topic: { slug: string, keyword: string, feedCount: number }, + * topic: import('../../../../../lib/topicSet.js').ResolvedTopic, * group: NonNullable>, * }|null>} */ @@ -34,12 +34,7 @@ async function resolve({ params }) { if (!group?.player) return null; const client = db(); - const slug = slugFromUrl(keyword); - - const [topic, counts] = await Promise.all([ - q.topicBySlug(client, slug), - q.topicKindCounts(client, slug), - ]); + const { topic, counts } = await resolveTopicsWithCounts(client, keyword); if (!topic) return null; // An empty sub-group is a 404 here for the same reason it is one on the @@ -59,7 +54,7 @@ export async function generateMetadata(props) { if (!found) return { title: 'Not found' }; const { topic, group } = found; - const path = `/topics/${encodeURIComponent(topic.slug)}/${group.segment}/play`; + const path = `/topics/${topicSegment(topic.slug)}/${group.segment}/play`; return { title: `${topic.keyword}: ${group.heading.toLowerCase()} — playing`, diff --git a/apps/web/src/app/topics/[keyword]/page.jsx b/apps/web/src/app/topics/[keyword]/page.jsx index 34281099..b137ac0c 100644 --- a/apps/web/src/app/topics/[keyword]/page.jsx +++ b/apps/web/src/app/topics/[keyword]/page.jsx @@ -1,9 +1,8 @@ import { notFound } from 'next/navigation'; -import { q } from '@rssamplifier/db'; import { db, siteUrl } from '../../../lib/db.js'; import { feedAlternates } from '../../../lib/subscribe.js'; -import { slugFromUrl } from '../../../lib/topicGroups.js'; +import { resolveTopics, resolveTopicsWithCounts, topicSegment } from '../../../lib/topicSet.js'; import { pageNumber } from '../../CategoryIndex.jsx'; import TopicListing from './TopicListing.jsx'; @@ -14,14 +13,19 @@ export const dynamic = 'force-dynamic'; */ export async function generateMetadata({ params }) { const { keyword } = await params; - const topic = await q.topicBySlug(db(), slugFromUrl(keyword)); + const { topic } = await resolveTopics(db(), keyword); if (!topic) return { title: 'Not found' }; - const page = `${siteUrl()}/topics/${encodeURIComponent(topic.slug)}`; + const page = `${siteUrl()}/topics/${topicSegment(topic.slug)}`; return { title: topic.keyword, description: `${topic.feedCount} blogs and podcasts in the RSS Amplifier directory write about ${topic.keyword}.`, + // A combined view is every listed topic's feeds again, in another order, + // under one of a combinatorial number of URLs. Canonical to itself (it is + // its own page, not one of its topics) and out of the index, the same way + // the player pages are; the links on it still count. + ...(topic.multi ? { robots: { index: false, follow: true } } : {}), alternates: { canonical: page, // Autodiscovery, which is how a browser extension or a reader offers to @@ -41,13 +45,9 @@ export async function generateMetadata({ params }) { */ export default async function TopicPage({ params, searchParams }) { const { keyword } = await params; - const slug = slugFromUrl(keyword); const client = db(); - const [topic, counts] = await Promise.all([ - q.topicBySlug(client, slug), - q.topicKindCounts(client, slug), - ]); + const { topic, counts } = await resolveTopicsWithCounts(client, keyword); if (!topic) notFound(); return ( diff --git a/apps/web/src/app/topics/[keyword]/play/page.jsx b/apps/web/src/app/topics/[keyword]/play/page.jsx index c101aca6..604298d4 100644 --- a/apps/web/src/app/topics/[keyword]/play/page.jsx +++ b/apps/web/src/app/topics/[keyword]/play/page.jsx @@ -1,8 +1,7 @@ import { notFound } from 'next/navigation'; -import { q } from '@rssamplifier/db'; import { db, siteUrl } from '../../../../lib/db.js'; -import { slugFromUrl } from '../../../../lib/topicGroups.js'; +import { resolveTopics, topicSegment } from '../../../../lib/topicSet.js'; import TopicPlayer from '../TopicPlayer.jsx'; export const dynamic = 'force-dynamic'; @@ -19,7 +18,7 @@ export const dynamic = 'force-dynamic'; */ async function resolve({ params }) { const { keyword } = await params; - const topic = await q.topicBySlug(db(), slugFromUrl(keyword)); + const { topic } = await resolveTopics(db(), keyword); return topic ? { topic } : null; } @@ -36,7 +35,7 @@ export async function generateMetadata(props) { return { title: `${topic.keyword} — playing`, description: `Play the recent podcast episodes and tracks on ${topic.keyword} in your browser.`, - alternates: { canonical: `${siteUrl()}/topics/${encodeURIComponent(topic.slug)}/play` }, + alternates: { canonical: `${siteUrl()}/topics/${topicSegment(topic.slug)}/play` }, // Deliberately out of the index. This page is the topic's own listing with // a transport on it, and there is one per topic and per category of one — // tens of thousands of pages whose text is a heading and a list of episode diff --git a/apps/web/src/lib/llms.js b/apps/web/src/lib/llms.js index 783080c9..79311497 100644 --- a/apps/web/src/lib/llms.js +++ b/apps/web/src/lib/llms.js @@ -166,6 +166,7 @@ export async function llmsTxt(opts = {}) { `- [Search, JSON](${base}/api/search?q=): full-text over posts and blogs; add &mode=any to match any term rather than all of them`, `- [Topics, JSON](${base}/api/topics): shared topics; ?q= searches them, ?min=10 for the well-covered ones`, `- [One topic, JSON](${base}/api/topics/{keyword}): the feeds filed under it, plus which categories it has; \`?group=\` narrows it to one`, + `- [Several topics, JSON](${base}/api/topics/business,saas,crypto): up to 10, comma-separated, as one union (each feed once) in the same shape; the same list works on /topics/{a},{b}, its sub-groups and its .rss/.atom/.json feeds`, `- [One topic, as a feed](${base}/topics/{keyword}.json): what those feeds published — also .rss, .atom, .m3u, .pls`, `- [Authors, JSON](${base}/api/authors): the people behind the feeds and where else they publish; ?network=email|fediverse|bluesky|github|website|linktree, ?q= searches names, ?min= sets the confidence floor`, `- [One author, JSON](${base}/api/authors/{slug}): their links and everything they publish here; ?feed= on /api/authors finds the people behind one feed`, diff --git a/apps/web/src/lib/mcp/tools.js b/apps/web/src/lib/mcp/tools.js index b3ff64e0..03b448e0 100644 --- a/apps/web/src/lib/mcp/tools.js +++ b/apps/web/src/lib/mcp/tools.js @@ -17,6 +17,7 @@ import { clampRawInput } from '../submitted.js'; import { clip, plainText } from './text.js'; import { WEBHOOK_TOOLS } from './webhookTools.js'; import { clientIp } from '@profullstack/x402-gateway/edge'; +import { resolveTopics, topicQueryArg, topicSegment } from '../topicSet.js'; /** * What an agent can do with the directory. @@ -285,11 +286,14 @@ export const TOOLS = [ name: 'get_topic', title: 'Feeds covering a topic', description: - 'Every feed filed under one topic. The keyword is normalised, so "Home Lab", "home lab" and "home-lab" all reach the same topic and you can pass a phrase you read anywhere. Each feed says whether it landed here from the publisher\'s own tag or from a phrase counted across its writing.', + 'Every feed filed under one topic. The keyword is normalised, so "Home Lab", "home lab" and "home-lab" all reach the same topic and you can pass a phrase you read anywhere. Each feed says whether it landed here from the publisher\'s own tag or from a phrase counted across its writing. A comma-separated list ("business, saas, crypto", up to 10) returns the union: every feed filed under any of them, each once, with `topics` naming the known ones and `unknown` the rest.', inputSchema: { type: 'object', properties: { - keyword: { type: 'string', description: 'The topic, in any spelling.' }, + keyword: { + type: 'string', + description: 'The topic, in any spelling — or several, separated by commas.', + }, limit: { type: 'integer', minimum: 1, maximum: 500, description: 'Default 100.' }, offset: { type: 'integer', minimum: 0, description: 'Default 0.' }, }, @@ -297,28 +301,25 @@ export const TOOLS = [ }, annotations: { readOnlyHint: true, openWorldHint: true }, async run(args) { - const slug = topicSlug(String(args?.keyword ?? '')); - if (!slug) throw invalid('keyword is required'); - - const client = db(); - const topic = await q.topicBySlug(client, slug); - if (!topic) throw invalid(`no topic '${args?.keyword}' — try list_topics or search`); + const topic = await mcpTopic(args?.keyword); const limit = bounded(args?.limit, 100, 1, 500); const offset = Math.max(Number(args?.offset ?? 0) || 0, 0); - const rows = await q.feedsForTopic(client, slug, { limit, offset }); + const rows = await q.feedsForTopic(db(), topicQueryArg(topic), { limit, offset }); return { slug: topic.slug, keyword: topic.keyword, total: topic.feedCount, + ...topicSetFields(topic), limit, offset, - page: `${siteUrl()}/topics/${encodeURIComponent(topic.slug)}`, + page: `${siteUrl()}/topics/${topicSegment(topic.slug)}`, feeds: rows.map((f) => ({ ...feed(f), source: f.source, strength: Number(f.count ?? 0), + ...(topic.multi ? { topics: String(f.topics ?? '').split(',').filter(Boolean) } : {}), })), }; }, @@ -328,32 +329,31 @@ export const TOOLS = [ name: 'topic_posts', title: 'Recent posts on a topic', description: - 'What the feeds covering a topic have actually published lately, newest first — the river rather than the roster. Note the difference from get_topic, which lists who covers the subject; this lists what they wrote. Drawn from the feeds most strongly filed under the topic, so a very broad keyword is a sample rather than a census.', + 'What the feeds covering a topic have actually published lately, newest first — the river rather than the roster. Note the difference from get_topic, which lists who covers the subject; this lists what they wrote. Drawn from the feeds most strongly filed under the topic, so a very broad keyword is a sample rather than a census. A comma-separated list ("business, saas, crypto", up to 10) is one river across all of them.', inputSchema: { type: 'object', properties: { - keyword: { type: 'string', description: 'The topic, in any spelling.' }, + keyword: { + type: 'string', + description: 'The topic, in any spelling — or several, separated by commas.', + }, limit: { type: 'integer', minimum: 1, maximum: 200, description: 'Default 50.' }, }, required: ['keyword'], }, annotations: { readOnlyHint: true, openWorldHint: true }, async run(args) { - const slug = topicSlug(String(args?.keyword ?? '')); - if (!slug) throw invalid('keyword is required'); - - const client = db(); - const topic = await q.topicBySlug(client, slug); - if (!topic) throw invalid(`no topic '${args?.keyword}' — try list_topics or search`); + const topic = await mcpTopic(args?.keyword); const limit = bounded(args?.limit, 50, 1, 200); - const rows = await q.itemsForTopic(client, slug, { limit }); + const rows = await q.itemsForTopic(db(), topicQueryArg(topic), { limit }); return { slug: topic.slug, keyword: topic.keyword, + ...topicSetFields(topic), limit, - feed: `${siteUrl()}/topics/${encodeURIComponent(topic.slug)}.rss`, + feed: `${siteUrl()}/topics/${topicSegment(topic.slug)}.rss`, posts: rows.map(post), }; }, @@ -1025,6 +1025,52 @@ function boundedFloat(raw, fallback, min, max) { return Math.min(Math.max(n, min), max); } +/** + * The topic a tool's `keyword` names: one, in any spelling, or a comma list. + * + * A keyword without a comma resolves exactly as before (topicSlug, then + * topicBySlug). A list goes through the same resolver the pages and the JSON + * endpoint use, so `get_topic` with "business, saas" is /api/topics/business,saas. + * + * @param {unknown} keyword + * @returns {Promise} + */ +async function mcpTopic(keyword) { + const raw = String(keyword ?? ''); + + if (!raw.includes(',')) { + const slug = topicSlug(raw); + if (!slug) throw invalid('keyword is required'); + const topic = await q.topicBySlug(db(), slug); + if (!topic) throw invalid(`no topic '${raw}' — try list_topics or search`); + return topic; + } + + const found = await resolveTopics(db(), encodeURIComponent(raw)); + if (found.error === 'too-many') throw invalid(`at most ${found.max} topics at once`); + if (!found.topic) throw invalid(`no topic in '${raw}' — try list_topics or search`); + return found.topic; +} + +/** + * The fields a combined topic adds to a tool's answer; nothing for one topic. + * + * @param {import('../topicSet.js').ResolvedTopic} topic + * @returns {Record} + */ +function topicSetFields(topic) { + if (!topic.multi) return {}; + return { + topics: (topic.topics ?? []).map((t) => ({ + slug: t.slug, + keyword: t.keyword, + feedCount: t.feedCount, + page: `${siteUrl()}/topics/${encodeURIComponent(t.slug)}`, + })), + unknown: topic.unknown ?? [], + }; +} + /** * A failure the caller can fix, as opposed to one we should be told about. * diff --git a/apps/web/src/lib/player.js b/apps/web/src/lib/player.js index 1a335904..dcc2a906 100644 --- a/apps/web/src/lib/player.js +++ b/apps/web/src/lib/player.js @@ -28,12 +28,16 @@ const RAW_PARAM = 'dl'; * not to read a directory of feeds. It is also the redirect target below, and a * redirect wants somewhere honest to land. * - * @param {string} slug the topic slug, unencoded + * @param {string} slug the topic slug, unencoded; or several joined by commas * @param {string|null} [segment] the sub-group segment, if this is one category * @returns {string} */ export function playerPath(slug, segment = null) { - const topic = `/topics/${encodeURIComponent(slug)}`; + // Commas left readable, so a combined view (business,saas) plays at the + // address it is browsed at. A single slug never contains one. + // (Inline rather than topicSet.js's topicSegment: this module is imported by + // a client component, and topicSet pulls in the database package.) + const topic = `/topics/${String(slug).split(',').map(encodeURIComponent).join(',')}`; return segment ? `${topic}/${segment}/play` : `${topic}/play`; } diff --git a/apps/web/src/lib/topicFeed.js b/apps/web/src/lib/topicFeed.js index 7cbe5049..1342c163 100644 --- a/apps/web/src/lib/topicFeed.js +++ b/apps/web/src/lib/topicFeed.js @@ -10,6 +10,7 @@ import { db, siteUrl } from './db.js'; import { fetchFeedAds } from './feedAds.js'; import { playerPath, wantsPlayer } from './player.js'; import { slugFromUrl, topicGroup } from './topicGroups.js'; +import { parseTopicList, resolveTopics, topicQueryArg, topicSegment } from './topicSet.js'; /** How many items a topic feed carries by default, and at most. */ const DEFAULT_LIMIT = 50; @@ -48,6 +49,10 @@ export const PLAYLIST_LIMIT = DEFAULT_LIMIT; * different questions. A reader subscribing to `/topics/physics.rss` wants to be * told when somebody writes about physics, not to be handed a directory. * + * Several topics — `/topics/business,saas,crypto.rss` — are one river drawn + * from all of them, newest first, each topic contributing its own strongest + * feeds (see pickedTopicFeeds in the db package for the cap). + * * @param {{ * keyword: string, * format: string, @@ -102,17 +107,29 @@ export async function topicFeed({ // being given are genuinely different resources, and 303 is the status that // says so without implying the playlist has moved. if (spec.media && wantsPlayer(req)) { + const { slugs, list } = parseTopicList(keyword); return Response.redirect( - `${siteUrl()}${playerPath(slug, group?.segment ?? null)}`, + `${siteUrl()}${playerPath(list ? slugs.join(',') : slug, group?.segment ?? null)}`, 303, ); } const client = db(); - const topic = await q.topicBySlug(client, slug); + const found = await resolveTopics(client, keyword); + + if (found.error === 'too-many') { + return fail(format, 400, 'too many topics', `Combine at most ${found.max}.`); + } + + const topic = found.topic; if (!topic) { - return fail(format, 404, `no such topic: ${slug}`, `Browse ${siteUrl()}/topics`); + return fail( + format, + 404, + `no such topic: ${found.unknown.join(',') || slug}`, + `Browse ${siteUrl()}/topics`, + ); } // A playlist can only carry files, so it is drawn from a different query than @@ -120,12 +137,12 @@ export async function topicFeed({ const kinds = group?.kinds ?? null; const [rows, counts] = await Promise.all([ spec.media - ? q.mediaForTopic(client, slug, { limit, kinds }) - : q.itemsForTopic(client, slug, { limit, kinds }), - group ? q.topicKindCounts(client, slug) : Promise.resolve({}), + ? q.mediaForTopic(client, topicQueryArg(topic), { limit, kinds }) + : q.itemsForTopic(client, topicQueryArg(topic), { limit, kinds }), + group ? q.topicKindCounts(client, topicQueryArg(topic)) : Promise.resolve({}), ]); - const base = `${siteUrl()}/topics/${encodeURIComponent(topic.slug)}`; + const base = `${siteUrl()}/topics/${topicSegment(topic.slug)}`; const page = group ? `${base}/${group.segment}` : base; const subject = group ? `${topic.keyword} (${group.noun})` : topic.keyword; diff --git a/apps/web/src/lib/topicSet.js b/apps/web/src/lib/topicSet.js new file mode 100644 index 00000000..a4dd5247 --- /dev/null +++ b/apps/web/src/lib/topicSet.js @@ -0,0 +1,200 @@ +import { q } from '@rssamplifier/db'; + +import { slugFromUrl } from './topicGroups.js'; + +/** + * Several topics as one: /topics/business,saas,crypto. + * + * A comma in a topic URL means a list. No topic slug can contain one — the + * slugger turns every run of punctuation into a hyphen — so a comma was never + * part of a working single-topic address, and reading it as a separator takes + * nothing away. Every topic surface (the page and its sub-groups, the JSON + * endpoint, the feeds and playlists, the player, the MCP tools) resolves its + * keyword through `resolveTopics` below, which is what keeps them one address. + * + * The combined view is the UNION of the listed topics: every feed filed under + * any of them, each feed once. Its order is the single-topic order applied + * across the set — a feed its publisher tagged with any of the topics first, + * then by its strength summed across the topics (see q.feedsForTopic) — so a + * feed that covers two of them rises above one that covers only one. + */ + +/** The most topics one combined view takes. More is a 400, not a truncation. */ +export const MAX_TOPICS = 10; + +/** + * Split a URL segment into topic slugs. + * + * Accepts the comma itself, `%2C` (which a client that encodes everything will + * send), and spaces around the items. Each item is normalised exactly as a + * single topic is, and repeats collapse, so `/topics/SaaS, saas,crypto` is two + * topics, in the order first named. + * + * @param {unknown} raw the keyword segment, encoded or not + * @returns {{ slugs: string[], list: boolean }} `list` is whether a comma was + * present at all, which is what decides whether this is a combined view even + * when only one distinct topic is left in it + */ +export function parseTopicList(raw) { + const decoded = safeDecode(String(raw ?? '')); + if (!decoded.includes(',')) { + const slug = slugFromUrl(raw); + return { slugs: slug ? [slug] : [], list: false }; + } + + const slugs = [ + ...new Set( + decoded + .split(',') + .map((part) => slugFromUrl(encodeURIComponent(part.trim()))) + .filter(Boolean), + ), + ]; + return { slugs, list: true }; +} + +/** + * A topic path segment, commas left readable. + * + * `encodeURIComponent` would write /topics/business%2Csaas — a working address, + * but not the one anybody typed, and not the one a combined view should + * canonicalise to. + * + * @param {string} slug one slug, or several joined by commas + * @returns {string} + */ +export function topicSegment(slug) { + return String(slug) + .split(',') + .map((part) => encodeURIComponent(part)) + .join(','); +} + +/** + * @typedef {{ + * slug: string, + * keyword: string, + * feedCount: number, + * multi?: boolean, + * slugs?: string[], + * topics?: Array<{ slug: string, keyword: string, feedCount: number }>, + * unknown?: string[], + * }} ResolvedTopic + */ + +/** + * The topic, or combined topics, a URL segment names. + * + * A single keyword resolves exactly as it always has, through q.topicBySlug. + * + * A list resolves to a synthetic topic whose `slug` is the known slugs joined + * by commas (the canonical address), whose `keyword` is their names joined by + * ", ", and whose `feedCount` is the union's distinct feed count. `slugs` is + * what to hand the queries — every topic query takes an array for the union — + * and `unknown` lists what was asked for and does not exist, so a page can say + * so rather than silently showing less. + * + * Outcomes: + * - `{ topic }` — something to show; + * - `{ error: 'not-found', unknown }` — nothing named exists (404); + * - `{ error: 'too-many', max }` — the list is longer than MAX_TOPICS (400). + * + * One exception for a list where nothing is known: the whole segment is tried + * once more as a single topic, so a keyword that was always spelled with a + * comma ("1,000 words" slugs to 1-000-words) keeps the page it had. + * + * @param {import('@rssamplifier/db').Client} client + * @param {unknown} raw + * @returns {Promise< + * | { topic: ResolvedTopic, error?: undefined } + * | { topic?: undefined, error: 'not-found', unknown: string[] } + * | { topic?: undefined, error: 'too-many', max: number } + * >} + */ +export async function resolveTopics(client, raw) { + const { slugs, list } = parseTopicList(raw); + + if (!list) { + const topic = slugs[0] ? await q.topicBySlug(client, slugs[0]) : null; + return topic ? { topic } : { error: 'not-found', unknown: slugs }; + } + + if (slugs.length > MAX_TOPICS) return { error: 'too-many', max: MAX_TOPICS }; + + const { topics, feedCount } = await q.topicsBySlugs(client, slugs); + + if (topics.length === 0) { + const whole = slugFromUrl(raw); + const single = whole ? await q.topicBySlug(client, whole) : null; + return single ? { topic: single } : { error: 'not-found', unknown: slugs }; + } + + const known = new Set(topics.map((t) => t.slug)); + const unknown = slugs.filter((slug) => !known.has(slug)); + + // `/topics/business,` or `/topics/business,Business` is one topic, and is + // that topic's own page rather than a second copy of it under another URL. + if (topics.length === 1 && unknown.length === 0) return { topic: topics[0] }; + + return { + topic: { + slug: topics.map((t) => t.slug).join(','), + keyword: topics.map((t) => t.keyword).join(', '), + feedCount, + multi: true, + slugs: topics.map((t) => t.slug), + topics, + unknown, + }, + }; +} + +/** + * resolveTopics plus the per-category counts every listing needs. + * + * A single topic asks both at once, as the pages always have; a list has to + * know which of its topics exist before it can count them. + * + * @param {import('@rssamplifier/db').Client} client + * @param {unknown} raw + * @returns {Promise<{ topic: ResolvedTopic|null, counts: Record }>} + */ +export async function resolveTopicsWithCounts(client, raw) { + const { slugs, list } = parseTopicList(raw); + + if (!list) { + const [found, counts] = await Promise.all([ + resolveTopics(client, raw), + q.topicKindCounts(client, slugs[0] ?? ''), + ]); + return { topic: found.topic ?? null, counts }; + } + + const found = await resolveTopics(client, raw); + if (!found.topic) return { topic: null, counts: {} }; + return { topic: found.topic, counts: await q.topicKindCounts(client, topicQueryArg(found.topic)) }; +} + +/** + * What to hand a topic query for this topic: its slug, or the union's slugs. + * + * @param {ResolvedTopic} topic + * @returns {string|string[]} + */ +export function topicQueryArg(topic) { + return topic.multi && topic.slugs ? topic.slugs : topic.slug; +} + +/** + * decodeURIComponent that leaves a malformed `%` alone instead of throwing. + * + * @param {string} value + * @returns {string} + */ +function safeDecode(value) { + try { + return decodeURIComponent(value); + } catch { + return value; + } +} diff --git a/apps/web/test/topic-set.test.js b/apps/web/test/topic-set.test.js new file mode 100644 index 00000000..3424f0f7 --- /dev/null +++ b/apps/web/test/topic-set.test.js @@ -0,0 +1,204 @@ +import assert from 'node:assert/strict'; +import { test, before, after } from 'node:test'; + +import { nowIso, q } from '@rssamplifier/db'; +import { connectTest } from '@rssamplifier/db/src/testdb.js'; +import { topicSlug } from '@rssamplifier/feed'; + +import { MAX_TOPICS, parseTopicList, topicSegment } from '../src/lib/topicSet.js'; +import { playerPath } from '../src/lib/player.js'; + +/** + * Combined topic views: /topics/business,saas,crypto and every surface that + * takes a topic. The pure parsing is tested on its own; the rest runs the real + * route handlers and MCP tools against a throwaway database, because the + * promise being tested is that one topic answers exactly as it always did and + * a list answers in the same shape. + */ +let db; +let lib; +let apiRoute; +let feedRoute; +let tools; + +before(async () => { + db = await connectTest(); + const url = new URL( + process.env['TEST_DATABASE_URL'] ?? 'postgres://postgres:postgres@127.0.0.1:5439/postgres', + ); + url.pathname = `/${db.databaseName}`; + process.env['DATABASE_URL'] = url.toString(); + // No sponsored items in a feed under test: they come from the network. + process.env['FEED_ADS'] = '0'; + + for (const [slug, topics] of [ + ['biz', [['business', 5, 'category']]], + ['both', [['business', 2, 'content'], ['saas', 2, 'content']]], + ['saas-only', [['saas', 3, 'content']]], + ['coins', [['crypto', 1, 'content']]], + ]) { + const { id } = await q.insertFeed(db, { + slug, + feed_url: `https://${slug}.example/feed.xml`, + title: slug, + kind: 'blog', + }); + await q.replaceFeedKeywords( + db, + id, + // Slugged as the crawler slugs them: "saas" is stored as "saa", exactly as + // in production, which is why every expectation below says saa. + topics.map(([topic, count, source]) => ({ slug: topicSlug(topic), keyword: topic, words: 1, count, source })), + ); + await q.upsertItems(db, id, [ + { guid: `${slug}-1`, title: `${slug} post`, url: `https://${slug}.example/1`, publishedAt: nowIso() }, + ]); + } + + lib = await import('../src/lib/topicSet.js'); + apiRoute = await import('../src/app/api/topics/[keyword]/route.js'); + feedRoute = await import('../src/app/api/topics/[keyword]/feed/[format]/route.js'); + ({ TOOLS_BY_NAME: tools } = await import('../src/lib/mcp/tools.js')); +}); + +after(async () => { + const { db: shared } = await import('../src/lib/db.js'); + shared().close(); + db?.close(); +}); + +const api = async (keyword, query = '') => { + const res = await apiRoute.GET(new Request(`https://rssamplifier.com/api/topics/${keyword}${query}`), { + params: Promise.resolve({ keyword }), + }); + return { status: res.status, body: await res.json() }; +}; + +test('a comma, %2C and spaces all separate topics; repeats collapse', () => { + assert.deepEqual(parseTopicList('business,saas,crypto'), { + slugs: ['business', 'saa', 'crypto'], + list: true, + }); + assert.deepEqual(parseTopicList('business%2Csaas%2C%20Crypto'), { + slugs: ['business', 'saa', 'crypto'], + list: true, + }); + assert.deepEqual(parseTopicList('SaaS, saas ,,business').slugs, ['saa', 'business']); +}); + +test('a keyword without a comma is the single topic, normalised as before', () => { + assert.deepEqual(parseTopicList('Home%20Lab'), { slugs: ['home-lab'], list: false }); +}); + +test('a combined address keeps its commas readable', () => { + assert.equal(topicSegment('business,saas'), 'business,saas'); + assert.equal(topicSegment('home-lab'), 'home-lab'); + assert.equal(playerPath('business,saas', 'podcasts'), '/topics/business,saas/podcasts/play'); + assert.equal(playerPath('ai'), '/topics/ai/play'); +}); + +test('a single topic answers exactly as it did, with no combined-view fields', async () => { + const { status, body } = await api('business'); + assert.equal(status, 200); + assert.equal(body.slug, 'business'); + assert.equal(body.total, 2); + assert.equal('topics' in body, false); + assert.equal('unknown' in body, false); + assert.deepEqual( + body.feeds.map((f) => f.slug), + ['biz', 'both'], + ); + assert.equal('topics' in body.feeds[0], false); +}); + +test('a list answers in the same shape, for the union, each feed once', async () => { + const { status, body } = await api('business,saas,crypto', '?limit=10'); + assert.equal(status, 200); + assert.equal(body.slug, 'business,saa,crypto'); + assert.equal(body.keyword, 'business, saas, crypto'); + // 2 + 2 + 1 feeds, `both` counted once. + assert.equal(body.total, 4); + assert.equal(body.page, 'https://rssamplifier.com/topics/business,saa,crypto'); + assert.deepEqual( + body.topics.map((t) => [t.slug, t.feedCount]), + [ + ['business', 2], + ['saa', 2], + ['crypto', 1], + ], + ); + assert.deepEqual(body.unknown, []); + assert.deepEqual( + body.feeds.map((f) => f.slug), + ['biz', 'both', 'saas-only', 'coins'], + ); + assert.deepEqual(body.feeds[1].topics, ['business', 'saa']); + assert.equal(body.feeds[1].strength, 4); +}); + +test('limit, offset and group work on a list as on one topic', async () => { + const { body } = await api('business,saas', '?limit=1&offset=1&group=blogs'); + assert.equal(body.group, 'blogs'); + assert.equal(body.total, 3); + assert.deepEqual( + body.feeds.map((f) => f.slug), + ['both'], + ); +}); + +test('unknown topics in a list are named, and the known ones still answer', async () => { + const { status, body } = await api('business,nope'); + assert.equal(status, 200); + assert.deepEqual(body.unknown, ['nope']); + assert.equal(body.total, 2); +}); + +test('a list of nothing known is a 404; a list too long is a 400', async () => { + const missing = await api('nope,nada'); + assert.equal(missing.status, 404); + assert.equal(missing.body.slug, 'nope,nada'); + + const many = Array.from({ length: MAX_TOPICS + 1 }, (_, i) => `t${i}`).join(','); + const tooMany = await api(many); + assert.equal(tooMany.status, 400); + assert.equal(tooMany.body.error, 'too-many-topics'); +}); + +test('a list that names one topic is that topic, not a copy of it', async () => { + const { topic } = await lib.resolveTopics(db, 'business,Business'); + assert.equal(topic.slug, 'business'); + assert.equal(topic.multi, undefined); +}); + +test('the feed of a list is one river across all of its topics', async () => { + const res = await feedRoute.GET( + new Request('https://rssamplifier.com/api/topics/business,saas,crypto/feed/json'), + { params: Promise.resolve({ keyword: 'business,saas,crypto', format: 'json' }) }, + ); + assert.equal(res.status, 200); + const doc = await res.json(); + assert.deepEqual( + doc.items.map((i) => i.id).sort(), + ['biz-1', 'both-1', 'coins-1', 'saas-only-1'], + ); + assert.match(doc.home_page_url, /\/topics\/business,saa,crypto$/); +}); + +test('the MCP topic tools take a comma list', async () => { + const listed = await tools.get('get_topic').run({ keyword: 'business, saas' }); + assert.equal(listed.total, 3); + assert.deepEqual( + listed.topics.map((t) => t.slug), + ['business', 'saa'], + ); + + const one = await tools.get('get_topic').run({ keyword: 'business' }); + assert.equal(one.total, 2); + assert.equal('topics' in one, false); + + const posts = await tools.get('topic_posts').run({ keyword: 'saas,crypto' }); + assert.deepEqual( + posts.posts.map((p) => p.feed).sort(), + ['both', 'coins', 'saas-only'], + ); +}); diff --git a/package.json b/package.json index 8f58b905..645a81bc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rssamplifier.com", - "version": "0.2.4", + "version": "0.3.0", "private": true, "type": "module", "description": "Agent-first public RSS directory. Submit a URL, a list or an OPML file; every blog gets its own page.", diff --git a/packages/db/src/queries.js b/packages/db/src/queries.js index 2d6093c7..3dd61236 100644 --- a/packages/db/src/queries.js +++ b/packages/db/src/queries.js @@ -1243,6 +1243,91 @@ export async function topicBySlug(db, slug) { }; } +/** + * A topic argument as a list of distinct slugs. + * + * Every topic query takes either one slug (a string) or several (an array), + * the second being a combined view such as /topics/business,saas,crypto. A + * one-element array is the single topic, so the caller never has to special-case + * a list that happened to have one known topic left in it. + * + * @param {string|string[]} slug + * @returns {string[]} + */ +export function topicSlugList(slug) { + const list = Array.isArray(slug) ? slug : [slug]; + return [...new Set(list.map((s) => String(s ?? '')).filter(Boolean))]; +} + +/** + * A `k.slug in (…)` fragment for a list of two or more topics. + * + * Placeholders for the same reason as kindFilter: the slugs are normalised by + * the time they get here, but a list of URL segments is still user input. + * + * @param {string[]} slugs + * @param {string} [column] + * @returns {string} + */ +function slugIn(slugs, column = 'k.slug') { + return `${column} in (${slugs.map(() => '?').join(', ')})`; +} + +/** + * Which of several topics exist, with their names and sizes, plus how many + * distinct feeds the union of them covers. + * + * The combined topic views (/topics/a,b,c) need both: the per-topic rows to + * link each topic and to say which of the asked-for ones are unknown, and the + * union count because a feed filed under two of the topics is still one feed. + * Two statements in one round trip each, run side by side. + * + * @param {Client} db + * @param {string[]} slugs + * @returns {Promise<{ topics: Array<{ slug: string, keyword: string, feedCount: number }>, feedCount: number }>} + */ +export async function topicsBySlugs(db, slugs) { + const list = topicSlugList(slugs); + if (list.length === 0) return { topics: [], feedCount: 0 }; + + const [per, union] = await Promise.all([ + db.execute({ + sql: `select k.slug, + ${topicLabelSql('k.slug')} as keyword, + count(*) as feed_count + from feed_keywords k + join feeds f on f.id = k.feed_id and f.status <> 'dead' + where ${slugIn(list)} + group by k.slug`, + args: list, + }), + db.execute({ + sql: `select count(distinct k.feed_id) as n + from feed_keywords k + join feeds f on f.id = k.feed_id and f.status <> 'dead' + where ${slugIn(list)}`, + args: list, + }), + ]); + + const bySlug = new Map( + per.rows.map((row) => [ + String(row.slug), + { + slug: String(row.slug), + keyword: String(row.keyword ?? row.slug), + feedCount: Number(row.feed_count ?? 0), + }, + ]), + ); + + return { + // In the order they were asked for, which is the order the page names them. + topics: list.map((slug) => bySlug.get(slug)).filter(Boolean), + feedCount: Number(union.rows[0]?.n ?? 0), + }; +} + /** * The feeds filed under one topic, strongest first. * @@ -1253,13 +1338,47 @@ export async function topicBySlug(db, slug) { * rather than everything about physics. See `normalizeKinds`. * * @param {Client} db - * @param {string} slug + * @param {string|string[]} slug one topic, or several for their union * @param {{ limit?: number, offset?: number, kinds?: string[]|null }} [opts] * @returns {Promise} */ export async function feedsForTopic(db, slug, opts = {}) { const { limit = 60, offset = 0, kinds = null } = opts; const filter = kindFilter(normalizeKinds(kinds)); + const slugs = topicSlugList(slug); + + // Several topics at once: the union, each feed once. A feed's place in it is + // the single-topic rule applied across the set: tagged by its publisher under + // ANY of the topics ranks it with the tagged ones, and its strength is the + // SUM of its counts across the topics, so a feed that covers business and + // saas outranks one that covers only business as strongly. `topics` says + // which of the asked-for topics it was filed under; `keyword` is the spelling + // of its strongest one. + if (slugs.length > 1) { + const { rows } = await db.execute({ + sql: `with k as ( + select feed_id, + min(case source when 'category' then 0 else 1 end) as tagged, + sum(count) as count, + string_agg(slug, ',' order by slug) as topics, + (array_agg(keyword order by case source when 'category' then 0 else 1 end, + count desc, slug))[1] as keyword + from feed_keywords + where ${slugIn(slugs, 'slug')} + group by feed_id + ) + select f.slug, f.title, f.description, f.site_url, f.feed_url, f.category, f.item_count, + f.image_url, f.card_url, k.keyword, k.count, k.topics, + case k.tagged when 0 then 'category' else 'content' end as source + from k + join feeds f on f.id = k.feed_id + where f.status <> 'dead'${filter.sql} + order by k.tagged, k.count desc, f.title asc + limit ? offset ?`, + args: [...slugs, ...filter.args, limit, offset], + }); + return rows; + } const { rows } = await db.execute({ sql: `select f.slug, f.title, f.description, f.site_url, f.feed_url, f.category, f.item_count, @@ -1269,7 +1388,7 @@ export async function feedsForTopic(db, slug, opts = {}) { where k.slug = ? and f.status <> 'dead'${filter.sql} order by case k.source when 'category' then 0 else 1 end, k.count desc, f.title asc limit ? offset ?`, - args: [slug, ...filter.args, limit, offset], + args: [slugs[0] ?? '', ...filter.args, limit, offset], }); return rows; } @@ -1286,18 +1405,36 @@ export async function feedsForTopic(db, slug, opts = {}) { * "nothing here". * * @param {Client} db - * @param {string} slug + * @param {string|string[]} slug one topic, or several for their union * @returns {Promise>} */ export async function topicKindCounts(db, slug) { - const { rows } = await db.execute({ - sql: `select f.category, count(*) as n - from feed_keywords k - join feeds f on f.id = k.feed_id and f.status <> 'dead' - where k.slug = ? - group by f.category`, - args: [slug], - }); + const slugs = topicSlugList(slug); + + // For several topics a feed filed under two of them is counted once, so the + // groups add up to the union's feedCount from topicsBySlugs. `count(distinct)` + // over the slug-index join rather than `f.id in (subquery)`: measured on + // production (business,saas,crypto) the semi-join walked all 560k feeds by + // category index, 0.3-2 s; this drives from feed_keywords_slug_idx, ~0.1 s. + const { rows } = await db.execute( + slugs.length > 1 + ? { + sql: `select f.category, count(distinct f.id) as n + from feed_keywords k + join feeds f on f.id = k.feed_id and f.status <> 'dead' + where ${slugIn(slugs)} + group by f.category`, + args: slugs, + } + : { + sql: `select f.category, count(*) as n + from feed_keywords k + join feeds f on f.id = k.feed_id and f.status <> 'dead' + where k.slug = ? + group by f.category`, + args: [slugs[0] ?? ''], + }, + ); /** @type {Record} */ const counts = {}; @@ -1448,6 +1585,57 @@ export async function latestItems(db, opts = {}) { return rows; } +/** + * The `picked` CTE a topic river draws its feeds from, for one topic or several. + * + * One topic is the original statement, unchanged: its `feedCap` strongest feeds. + * + * Several topics are not "the strongest feedCap across the union", because one + * large topic would then fill the whole cap and a combined river of + * business,saas,crypto would be a business river. Each topic contributes its + * own strongest `perTopic` feeds instead, ranked by the same rule, and a feed + * strong in two of them is picked once. `perTopic` keeps the total near twice + * the single-topic cap whatever the list length (200 each for two topics, 134 + * for three, never fewer than 50), so the per-feed index seeks stay bounded. + * + * @param {string[]} slugs + * @param {{ sql: string, args: string[] }} filter + * @param {number} feedCap + * @returns {{ sql: string, args: unknown[] }} + */ +function pickedTopicFeeds(slugs, filter, feedCap) { + if (slugs.length <= 1) { + return { + sql: `picked as materialized ( + select k.feed_id from feed_keywords k + join feeds f on f.id = k.feed_id and f.status <> 'dead' + where k.slug = ?${filter.sql} + order by case k.source when 'category' then 0 else 1 end, k.count desc + limit ? + )`, + args: [slugs[0] ?? '', ...filter.args, feedCap], + }; + } + + const perTopic = Math.max(50, Math.ceil((feedCap * 2) / slugs.length)); + return { + sql: `ranked as ( + select k.feed_id, + row_number() over ( + partition by k.slug + order by case k.source when 'category' then 0 else 1 end, k.count desc + ) as rn + from feed_keywords k + join feeds f on f.id = k.feed_id and f.status <> 'dead' + where ${slugIn(slugs)}${filter.sql} + ), + picked as materialized ( + select distinct feed_id from ranked where rn <= ? + )`, + args: [...slugs, ...filter.args, Math.min(perTopic, feedCap)], + }; +} + /** * Recent posts from across a topic, newest first. * @@ -1462,7 +1650,7 @@ export async function latestItems(db, opts = {}) { * who wrote what without it. * * @param {Client} db - * @param {string} slug + * @param {string|string[]} slug one topic, or several for their union * @param {{ * limit?: number, * feedCap?: number, @@ -1498,14 +1686,10 @@ export async function itemsForTopic(db, slug, opts = {}) { // key would be run over every row it returns. const want = group ? Math.min(limit * 3, 600) : limit; + const picked = pickedTopicFeeds(topicSlugList(slug), filter, feedCap); + const { rows } = await db.execute({ - sql: `with picked as materialized ( - select k.feed_id from feed_keywords k - join feeds f on f.id = k.feed_id and f.status <> 'dead' - where k.slug = ?${filter.sql} - order by case k.source when 'category' then 0 else 1 end, k.count desc - limit ? - ) + sql: `with ${picked.sql} select i.guid, i.url, i.title, i.summary, i.author, i.image_url, i.published_at, i.audio_url, i.audio_type, i.audio_bytes, i.audio_seconds, i.cluster_key, f.slug as feed_slug, f.title as feed_title, f.feed_url, f.category, @@ -1532,7 +1716,7 @@ export async function itemsForTopic(db, slug, opts = {}) { limit ?`, // `filter.args` belongs to the `picked` subquery; `since`, the ceiling and the // first `want` to the per-feed range; the last `want` to the page. - args: [slug, ...filter.args, feedCap, since, riverCeiling(), want, want], + args: [...picked.args, since, riverCeiling(), want, want], }); if (!group) return rows; @@ -1554,7 +1738,7 @@ export async function itemsForTopic(db, slug, opts = {}) { * which cuts the row count enough to stay inside a couple of seconds. * * @param {Client} db - * @param {string} slug + * @param {string|string[]} slug one topic, or several for their union * @param {{ limit?: number, feedCap?: number }} [opts] * @returns {Promise} */ @@ -1562,14 +1746,10 @@ export async function mediaForTopic(db, slug, opts = {}) { const { limit = 100, feedCap = TOPIC_RIVER_FEEDS, kinds = null } = opts; const filter = kindFilter(normalizeKinds(kinds)); + const picked = pickedTopicFeeds(topicSlugList(slug), filter, feedCap); + const { rows } = await db.execute({ - sql: `with picked as materialized ( - select k.feed_id from feed_keywords k - join feeds f on f.id = k.feed_id and f.status <> 'dead' - where k.slug = ?${filter.sql} - order by case k.source when 'category' then 0 else 1 end, k.count desc - limit ? - ) + sql: `with ${picked.sql} select i.id as item_id, i.guid, i.url, i.title, i.summary, i.author, i.image_url, i.published_at, i.audio_url, i.audio_type, i.audio_bytes, i.audio_seconds, @@ -1584,7 +1764,7 @@ export async function mediaForTopic(db, slug, opts = {}) { join feeds f on f.id = i.feed_id order by i.published_at desc nulls last, i.created_at desc limit ?`, - args: [slug, ...filter.args, feedCap, limit, limit], + args: [...picked.args, limit, limit], }); return rows; } diff --git a/packages/db/test/multi-topic.test.js b/packages/db/test/multi-topic.test.js new file mode 100644 index 00000000..b529136b --- /dev/null +++ b/packages/db/test/multi-topic.test.js @@ -0,0 +1,152 @@ +import assert from 'node:assert/strict'; +import { test, before, after } from 'node:test'; + +import { nowIso } from '../src/client.js'; +import { connectTest } from '../src/testdb.js'; +import * as q from '../src/queries.js'; + +/** + * Three topics that overlap: /topics/business,saas,crypto. + * + * `both` is filed under business AND saas, which is the case the union exists + * for: it must come back once, counted once, and rank on its combined strength. + */ +let db; + +const FEEDS = [ + // slug, kind, [topic, count, source]... + ['biz-tagged', 'blog', [['business', 2, 'category']]], + ['biz-counted', 'blog', [['business', 9, 'content']]], + ['both', 'blog', [['business', 4, 'content'], ['saas', 4, 'content']]], + ['saas-show', 'podcast', [['saas', 3, 'category']]], + ['coin-blog', 'blog', [['crypto', 1, 'content']]], +]; + +before(async () => { + db = await connectTest(); + + for (const [slug, kind, topics] of FEEDS) { + const { id } = await q.insertFeed(db, { + slug, + feed_url: `https://${slug}.example/feed.xml`, + title: slug, + kind, + }); + + await q.replaceFeedKeywords( + db, + id, + topics.map(([topic, count, source]) => ({ + slug: topic, + keyword: topic, + words: 1, + count, + source, + })), + ); + + await q.upsertItems(db, id, [ + { + guid: `${slug}-1`, + title: `${slug} post`, + url: `https://${slug}.example/1`, + publishedAt: nowIso(), + audio: + kind === 'podcast' ? { url: `https://${slug}.example/1.mp3`, type: 'audio/mpeg' } : null, + }, + ]); + } +}); + +after(async () => { + db.close(); +}); + +test('a one-element list is the single topic, row for row', async () => { + assert.deepEqual( + await q.feedsForTopic(db, ['business']), + await q.feedsForTopic(db, 'business'), + ); + assert.deepEqual(await q.topicKindCounts(db, ['saas']), await q.topicKindCounts(db, 'saas')); + assert.deepEqual( + (await q.itemsForTopic(db, ['business'])).map((i) => i.guid), + (await q.itemsForTopic(db, 'business')).map((i) => i.guid), + ); +}); + +test('the single-topic ordering is unchanged: tagged first, then strength', async () => { + const rows = await q.feedsForTopic(db, 'business'); + assert.deepEqual( + rows.map((f) => String(f.slug)), + ['biz-tagged', 'biz-counted', 'both'], + ); +}); + +test('several topics list their union, each feed once', async () => { + const rows = await q.feedsForTopic(db, ['business', 'saas', 'crypto']); + const slugs = rows.map((f) => String(f.slug)); + + assert.equal(new Set(slugs).size, slugs.length); + assert.deepEqual([...slugs].sort(), ['biz-counted', 'biz-tagged', 'both', 'coin-blog', 'saas-show']); + + // Tagged under any topic first; then summed strength, so `both` (4 + 4) + // outranks nothing it should not and sits right behind biz-counted (9). + assert.deepEqual(slugs, ['saas-show', 'biz-tagged', 'biz-counted', 'both', 'coin-blog']); + + const both = rows.find((f) => f.slug === 'both'); + assert.equal(Number(both.count), 8); + assert.equal(both.topics, 'business,saas'); + assert.equal(rows.find((f) => f.slug === 'saas-show').source, 'category'); +}); + +test('the union pages and filters like a single topic', async () => { + const page2 = await q.feedsForTopic(db, ['business', 'saas', 'crypto'], { limit: 2, offset: 2 }); + assert.deepEqual( + page2.map((f) => String(f.slug)), + ['biz-counted', 'both'], + ); + + const podcasts = await q.feedsForTopic(db, ['business', 'saas'], { kinds: ['podcast'] }); + assert.deepEqual( + podcasts.map((f) => String(f.slug)), + ['saas-show'], + ); +}); + +test('the union counts a shared feed once, per category and in total', async () => { + assert.deepEqual(await q.topicKindCounts(db, ['business', 'saas', 'crypto']), { + blog: 4, + podcast: 1, + }); + + const { topics, feedCount } = await q.topicsBySlugs(db, ['crypto', 'business', 'nope', 'saas']); + assert.equal(feedCount, 5); + // Known topics only, in the order asked for. + assert.deepEqual( + topics.map((t) => [t.slug, t.feedCount]), + [ + ['crypto', 1], + ['business', 3], + ['saas', 2], + ], + ); +}); + +test('the union river is every listed topic, newest first, once each', async () => { + const items = await q.itemsForTopic(db, ['business', 'saas', 'crypto'], { group: false }); + const guids = items.map((i) => String(i.guid)); + + assert.equal(new Set(guids).size, guids.length); + assert.deepEqual([...guids].sort(), FEEDS.map(([slug]) => `${slug}-1`).sort()); + + const media = await q.mediaForTopic(db, ['business', 'saas']); + assert.deepEqual( + media.map((i) => String(i.guid)), + ['saas-show-1'], + ); +}); + +test('a topic list is deduplicated and blanks are dropped', () => { + assert.deepEqual(q.topicSlugList(['a', 'b', 'a', '']), ['a', 'b']); + assert.deepEqual(q.topicSlugList('a'), ['a']); +});