feat(feeds): add curated RSS feed review

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+- Add user-curated RSS and Atom subscriptions with staged Review, Saved, and Dismissed collections.
+- Add source filtering, newest/oldest sorting, cursor-based infinite loading, and responsive feed management.
+- Add per-feed pause, future-only Auto-save, health warnings, and confirmed deletion that preserves saved articles.
+- Add autonomous conditional polling with failure backoff, bounded retention, SSRF-safe fetching, and parser workload limits.
 - Add admin dashboard with dedicated layout for account management and service health visibility.
 - Add admin account management: list, detail, update name/role, reset password, soft delete, and restore users.
 - Expose user role in current-user response; client role gates admin navigation affordance only.

README.md

@@ -34,6 +34,7 @@ Loreo is a read-it-later app for saving articles worth revisiting, built with se
 ### Saving & Organization
 
 - Save links with automatic article extraction
+- Subscribe to RSS/Atom feeds and review new items by source before saving
 - Favorite articles for quick access
 - Archive articles to hide them from your reading list
 - Organize with tag groups and tags
@@ -43,6 +44,7 @@ Loreo is a read-it-later app for saving articles worth revisiting, built with se
 ### Utilities
 
 - CSV import with field mapping
+- RSS feed polling through the existing Redis/BullMQ worker stack
 - Docker Compose support, run locally or as separate web/server processes
 
 ## Why Loreo?

apps/server/.env.example

@@ -34,6 +34,10 @@ RATE_LIMIT_MAX=50
 # Body size limit (in bytes)
 BODY_SIZE_LIMIT=102400
 
+# RSS polling scheduler
+FEED_POLL_SCAN_INTERVAL_MS=60000
+FEED_POLL_SCAN_BATCH_SIZE=100
+
 # Public/storage URLs
 PUBLIC_URL=http://localhost:3000
 STORAGE_PROVIDER=local

apps/server/scripts/migrate-test.ts

@@ -23,6 +23,7 @@ if (!env.DATABASE_DB.includes('test')) {
 
 console.log('Resetting test database schema...');
 await pool.query('DROP SCHEMA IF EXISTS public CASCADE;');
+await pool.query('DROP SCHEMA IF EXISTS drizzle CASCADE;');
 await pool.query('CREATE SCHEMA public;');
 console.log('Test database schema reset complete.');
 

apps/server/src/app.ts

@@ -5,13 +5,16 @@ import createApp from './lib/create-app.js';
 import type { Repos } from './lib/types.js';
 
 import { createDrizzleAuthAdapter } from './repositories/auth.repository.js';
+import { createDrizzleFeedItemsAdapter } from './repositories/feed-items.repository.js';
+import { createDrizzleFeedSubscriptionsAdapter } from './repositories/feed-subscriptions.repository.js';
 import { createDrizzleHighlightsAdapter } from './repositories/highlights.repository.js';
 import { createDrizzleImportSessionsAdapter } from './repositories/import-sessions.repository.js';
 import { createDrizzleLinksAdapter } from './repositories/links.repository.js';
 import { createDrizzleTagsAdapter } from './repositories/tags.repository.js';
 
 import admin from './routes/admin/admin.index.js';
 import auth from './routes/auth/auth.index.js';
+import feeds from './routes/feeds/feeds.index.js';
 import files from './routes/files/files.index.js';
 import health from './routes/health/health.index.js';
 import highlights from './routes/highlights/highlights.index.js';
@@ -24,6 +27,8 @@ const app = createApp();
 
 const repos: Repos = {
   auth: createDrizzleAuthAdapter(db),
+  feedItems: createDrizzleFeedItemsAdapter(db),
+  feedSubscriptions: createDrizzleFeedSubscriptionsAdapter(db),
   highlights: createDrizzleHighlightsAdapter(db),
   importSessions: createDrizzleImportSessionsAdapter(db),
   links: createDrizzleLinksAdapter(db),
@@ -43,6 +48,7 @@ const router = app
   .route('/', auth)
   .route('/', home)
   .route('/', links)
+  .route('/', feeds)
   .route('/', highlights)
   .route('/', tags)
   .route('/', files)

apps/server/src/db/migrations/0001_create_rss_feed_tables.sql

@@ -0,0 +1,57 @@
+CREATE TABLE "feed_items" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"subscription_id" uuid NOT NULL,
+	"user_id" uuid NOT NULL,
+	"link_id" uuid,
+	"guid" text,
+	"url" text NOT NULL,
+	"normalized_url" text NOT NULL,
+	"title" text NOT NULL,
+	"excerpt" text,
+	"author" text,
+	"published_at" timestamp with time zone,
+	"image_url" text,
+	"state" varchar(20) DEFAULT 'new' NOT NULL,
+	"discovered_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"saved_at" timestamp with time zone,
+	"dismissed_at" timestamp with time zone,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "feed_subscriptions" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"user_id" uuid NOT NULL,
+	"feed_url" text NOT NULL,
+	"normalized_feed_url" text NOT NULL,
+	"site_url" text,
+	"title" text NOT NULL,
+	"description" text,
+	"image_url" text,
+	"auto_save" boolean DEFAULT false NOT NULL,
+	"status" varchar(20) DEFAULT 'active' NOT NULL,
+	"last_fetched_at" timestamp with time zone,
+	"last_successful_fetch_at" timestamp with time zone,
+	"next_fetch_after" timestamp with time zone,
+	"last_error" text,
+	"failure_count" integer DEFAULT 0 NOT NULL,
+	"etag" text,
+	"last_modified" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+ALTER TABLE "feed_items" ADD CONSTRAINT "feed_items_subscription_id_feed_subscriptions_id_fk" FOREIGN KEY ("subscription_id") REFERENCES "public"."feed_subscriptions"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "feed_items" ADD CONSTRAINT "feed_items_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "feed_items" ADD CONSTRAINT "feed_items_link_id_links_id_fk" FOREIGN KEY ("link_id") REFERENCES "public"."links"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "feed_subscriptions" ADD CONSTRAINT "feed_subscriptions_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+CREATE UNIQUE INDEX "uq_feed_items_subscription_guid" ON "feed_items" USING btree ("subscription_id","guid");--> statement-breakpoint
+CREATE UNIQUE INDEX "uq_feed_items_subscription_normalized_url" ON "feed_items" USING btree ("subscription_id","normalized_url");--> statement-breakpoint
+CREATE INDEX "idx_feed_items_user_state_published" ON "feed_items" USING btree ("user_id","state","published_at" DESC NULLS LAST);--> statement-breakpoint
+CREATE INDEX "idx_feed_items_user_normalized_url" ON "feed_items" USING btree ("user_id","normalized_url");--> statement-breakpoint
+CREATE INDEX "idx_feed_items_subscription_discovered" ON "feed_items" USING btree ("subscription_id","discovered_at" DESC NULLS LAST);--> statement-breakpoint
+CREATE INDEX "idx_feed_items_link_id" ON "feed_items" USING btree ("link_id");--> statement-breakpoint
+CREATE UNIQUE INDEX "uq_feed_subscriptions_user_normalized_url" ON "feed_subscriptions" USING btree ("user_id","normalized_feed_url");--> statement-breakpoint
+CREATE INDEX "idx_feed_subscriptions_user_created" ON "feed_subscriptions" USING btree ("user_id","created_at" DESC NULLS LAST);--> statement-breakpoint
+CREATE INDEX "idx_feed_subscriptions_status_next_fetch" ON "feed_subscriptions" USING btree ("status","next_fetch_after");--> statement-breakpoint
+CREATE INDEX "idx_feed_subscriptions_user_status" ON "feed_subscriptions" USING btree ("user_id","status");

apps/server/src/db/migrations/0002_smooth_rss_feed_tables.sql

@@ -0,0 +1,86 @@
+CREATE TABLE IF NOT EXISTS "feed_subscriptions" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"user_id" uuid NOT NULL,
+	"feed_url" text NOT NULL,
+	"normalized_feed_url" text NOT NULL,
+	"site_url" text,
+	"title" text NOT NULL,
+	"description" text,
+	"image_url" text,
+	"auto_save" boolean DEFAULT false NOT NULL,
+	"status" varchar(20) DEFAULT 'active' NOT NULL,
+	"last_fetched_at" timestamp with time zone,
+	"last_successful_fetch_at" timestamp with time zone,
+	"next_fetch_after" timestamp with time zone,
+	"last_error" text,
+	"failure_count" integer DEFAULT 0 NOT NULL,
+	"etag" text,
+	"last_modified" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "feed_items" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"subscription_id" uuid NOT NULL,
+	"user_id" uuid NOT NULL,
+	"link_id" uuid,
+	"guid" text,
+	"url" text NOT NULL,
+	"normalized_url" text NOT NULL,
+	"title" text NOT NULL,
+	"excerpt" text,
+	"author" text,
+	"published_at" timestamp with time zone,
+	"image_url" text,
+	"state" varchar(20) DEFAULT 'new' NOT NULL,
+	"discovered_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"saved_at" timestamp with time zone,
+	"dismissed_at" timestamp with time zone,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+DO $$ BEGIN
+	ALTER TABLE "feed_subscriptions" ADD CONSTRAINT "feed_subscriptions_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;
+EXCEPTION
+	WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+DO $$ BEGIN
+	ALTER TABLE "feed_items" ADD CONSTRAINT "feed_items_subscription_id_feed_subscriptions_id_fk" FOREIGN KEY ("subscription_id") REFERENCES "public"."feed_subscriptions"("id") ON DELETE cascade ON UPDATE no action;
+EXCEPTION
+	WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+DO $$ BEGIN
+	ALTER TABLE "feed_items" ADD CONSTRAINT "feed_items_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;
+EXCEPTION
+	WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+DO $$ BEGIN
+	ALTER TABLE "feed_items" ADD CONSTRAINT "feed_items_link_id_links_id_fk" FOREIGN KEY ("link_id") REFERENCES "public"."links"("id") ON DELETE set null ON UPDATE no action;
+EXCEPTION
+	WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "uq_feed_subscriptions_user_normalized_url" ON "feed_subscriptions" USING btree ("user_id","normalized_feed_url");
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "idx_feed_subscriptions_user_created" ON "feed_subscriptions" USING btree ("user_id","created_at" DESC NULLS LAST);
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "idx_feed_subscriptions_status_next_fetch" ON "feed_subscriptions" USING btree ("status","next_fetch_after");
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "idx_feed_subscriptions_user_status" ON "feed_subscriptions" USING btree ("user_id","status");
+--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "uq_feed_items_subscription_guid" ON "feed_items" USING btree ("subscription_id","guid");
+--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "uq_feed_items_subscription_normalized_url" ON "feed_items" USING btree ("subscription_id","normalized_url");
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "idx_feed_items_user_state_published" ON "feed_items" USING btree ("user_id","state","published_at" DESC NULLS LAST);
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "idx_feed_items_user_normalized_url" ON "feed_items" USING btree ("user_id","normalized_url");
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "idx_feed_items_subscription_discovered" ON "feed_items" USING btree ("subscription_id","discovered_at" DESC NULLS LAST);
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "idx_feed_items_link_id" ON "feed_items" USING btree ("link_id");

apps/server/src/db/migrations/0003_feed_item_pagination_indexes.sql

@@ -0,0 +1,16 @@
+CREATE INDEX IF NOT EXISTS "idx_feed_items_user_state_effective_date_id"
+  ON "feed_items" USING btree (
+    "user_id",
+    "state",
+    (coalesce("published_at", "discovered_at")),
+    "id"
+  );
+
+CREATE INDEX IF NOT EXISTS "idx_feed_items_user_state_subscription_effective_date_id"
+  ON "feed_items" USING btree (
+    "user_id",
+    "state",
+    "subscription_id",
+    (coalesce("published_at", "discovered_at")),
+    "id"
+  );

apps/server/src/db/migrations/0004_feed_item_pagination_query_shapes.sql

@@ -0,0 +1,2 @@
+CREATE INDEX "idx_feed_items_user_effective_date_id" ON "feed_items" USING btree ("user_id",coalesce("published_at", "discovered_at"),"id");--> statement-breakpoint
+CREATE INDEX "idx_feed_items_user_subscription_effective_date_id" ON "feed_items" USING btree ("user_id","subscription_id",coalesce("published_at", "discovered_at"),"id");

apps/server/src/db/migrations/0005_feed_owner_constraints.sql

@@ -0,0 +1,6 @@
+CREATE UNIQUE INDEX "uq_feed_subscriptions_id_user" ON "feed_subscriptions" USING btree ("id","user_id");--> statement-breakpoint
+CREATE UNIQUE INDEX "uq_links_id_user" ON "links" USING btree ("id","user_id");--> statement-breakpoint
+ALTER TABLE "feed_items" ADD CONSTRAINT "fk_feed_items_subscription_owner" FOREIGN KEY ("subscription_id","user_id") REFERENCES "public"."feed_subscriptions"("id","user_id") ON DELETE cascade ON UPDATE no action NOT VALID;--> statement-breakpoint
+ALTER TABLE "feed_items" ADD CONSTRAINT "fk_feed_items_link_owner" FOREIGN KEY ("link_id","user_id") REFERENCES "public"."links"("id","user_id") ON DELETE no action ON UPDATE no action NOT VALID;--> statement-breakpoint
+ALTER TABLE "feed_items" ADD CONSTRAINT "chk_feed_items_state" CHECK ("feed_items"."state" in ('new', 'dismissed', 'saved')) NOT VALID;--> statement-breakpoint
+ALTER TABLE "feed_subscriptions" ADD CONSTRAINT "chk_feed_subscriptions_status" CHECK ("feed_subscriptions"."status" in ('active', 'paused')) NOT VALID;