PeerPush deals daily via @profullstack/scraper-daemon (1.10.0)

apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@c0upons/cli",
-  "version": "1.9.3",
+  "version": "1.10.0",
   "description": "c0upons CLI — search, submit, and manage coupons from the terminal",
   "type": "module",
   "bin": {

apps/cli/src/index.ts

@@ -158,6 +158,16 @@ program
       } catch {
         grocery.fail('Failed to reach c0upons.');
       }
+      const peerpush = ora('Reading the PeerPush deals board...').start();
+      try {
+        const res = await fetch(`${BASE_URL}/api/sync/peerpush`, { method: 'POST' });
+        const data = (await res.json()) as { error?: string; skipped?: boolean; deals?: number; inserted?: number; updated?: number; pruned?: number };
+        if (!res.ok) peerpush.fail(String(data.error ?? res.statusText));
+        else if (data.skipped) peerpush.info(chalk.gray('Read recently; skipped.'));
+        else peerpush.succeed(`${String(data.inserted)} new, ${String(data.updated)} updated of ${String(data.deals)} deals, ${String(data.pruned)} expired removed`);
+      } catch {
+        peerpush.fail('Failed to reach c0upons.');
+      }
     }
     if (!opts.syncOnly) {
       const spinner = ora('Reading deal pages for coupons without a code...').start();

apps/web/app/api/sync/peerpush/route.ts

@@ -0,0 +1,31 @@
+import { NextResponse } from 'next/server';
+import { getDb } from '@/lib/db';
+import { dbErrorResponse } from '@/lib/api-error';
+import { syncPeerpushDeals } from '@/lib/peerpush-sync';
+
+export const dynamic = 'force-dynamic';
+export const maxDuration = 60;
+
+/**
+ * Read peerpush.com/deals into stores and coupons now, instead of waiting for
+ * the daemon's daily run. Keyless like the other syncs: the deals are public
+ * and the write is an idempotent upsert; a run within an hour of the last one
+ * answers `skipped`, so a stranger cannot make us hammer PeerPush.
+ */
+async function run() {
+  try {
+    const result = await syncPeerpushDeals(getDb());
+    return NextResponse.json(result, { headers: { 'Cache-Control': 'no-store' } });
+  } catch (err) {
+    console.error('peerpush sync failed:', err);
+    return dbErrorResponse(err, 'peerpush sync failed');
+  }
+}
+
+export async function POST() {
+  return run();
+}
+
+export async function GET() {
+  return run();
+}

apps/web/app/api/v1/daemon/route.ts

@@ -0,0 +1,13 @@
+import { NextResponse } from 'next/server';
+import { getDaemon } from '@/lib/daemon';
+
+export const dynamic = 'force-dynamic';
+
+/** What the scraper daemon is doing: tasks, due work, failures, the last runs. Public, like every fleet daemon's. */
+export async function GET() {
+  const daemon = getDaemon();
+  if (!daemon) return NextResponse.json({ ok: false, running: false }, { status: 503, headers: { 'Cache-Control': 'no-store' } });
+  return NextResponse.json(await daemon.status(), {
+    headers: { 'Cache-Control': 'no-store', 'Access-Control-Allow-Origin': '*' },
+  });
+}

apps/web/app/docs/page.tsx

@@ -395,6 +395,20 @@ export default function DocsPage() {
             example="curl -X POST https://c0upons.com/api/sync/grocery"
             response={`{ "ok": true, "skipped": false, "listed": 212, "remaining": 209, "pruned": 0, "written": 331, "flyers": [{ "id": 8141217, "store": "Raley's", "postal_code": "95814", "items": 123, "written": 123 }] }`}
           />
+          <Endpoint
+            method="POST"
+            path="/api/sync/peerpush"
+            desc="Read peerpush.com/deals (discount codes software founders attach to their PeerPush listings) into stores and coupons, and remove deals past their valid-until date. The server's scraper daemon runs this daily; throttled to one run per hour."
+            example="curl -X POST https://c0upons.com/api/sync/peerpush"
+            response={`{ "ok": true, "skipped": false, "pages": 4, "deals": 127, "inserted": 127, "updated": 0, "declined": 0, "pruned": 0, "stores": 127, "added": ["Kliport: 50% OFF"] }`}
+          />
+          <Endpoint
+            method="GET"
+            path="/api/v1/daemon"
+            desc="What the scraper daemon (@profullstack/scraper-daemon) is doing: its jobs, the last run of each, failures and the next run."
+            example="curl https://c0upons.com/api/v1/daemon"
+            response={`{ "name": "c0upons", "engine": "bullmq", "running": true, "tasks": [{ "name": "peerpush", "every": 86400000, "ok": 1, "failed": 0, "lastError": null }] }`}
+          />
         </section>
 
         {/* MCP */}
@@ -423,6 +437,7 @@ curl -X POST https://c0upons.com/mcp -H 'content-type: application/json' \\
               ['sync_deals', 'Pull the next deals from nichedb.dev.'],
               ['sync_reddit', 'Read the newest r/couponcodes posts.'],
               ['sync_grocery', 'Read the next grocery weekly ads.'],
+              ['sync_peerpush', 'Read the PeerPush deals board.'],
               ['reveal_pending', 'Read the pages of up to three code-less coupons.'],
             ].map(([name, desc]) => (
               <div key={name} className="border border-gray-200 rounded-lg px-4 py-3">

apps/web/instrumentation.ts

@@ -1,7 +1,8 @@
 /**
  * Runs once when the Next.js server starts: starts the pollers that read
  * r/couponcodes every five minutes (lib/reddit-poller) and the grocery
- * weekly ads every five (lib/flipp-poller). The edge runtime also calls this
+ * weekly ads every five (lib/flipp-poller), and the scraper daemon (lib/daemon:
+ * peerpush.com/deals daily). The edge runtime also calls this
  * and has no timers or database, so it is skipped there.
  */
 export async function register() {
@@ -10,4 +11,6 @@ export async function register() {
   startRedditPoller();
   const { startFlippPoller } = await import('./lib/flipp-poller');
   startFlippPoller();
+  const { startDaemon } = await import('./lib/daemon');
+  startDaemon();
 }

apps/web/lib/daemon.ts

@@ -0,0 +1,98 @@
+import 'server-only';
+import { createDaemon, type Daemon } from '@profullstack/scraper-daemon';
+import { getDb } from './db';
+import { syncPeerpushDeals, USER_AGENT } from './peerpush-sync';
+
+/**
+ * c0upons' scraper daemon (@profullstack/scraper-daemon, the house updater for
+ * listing sites): BullMQ on c0upons' own Redis (REDIS_URL, the compose `redis`
+ * service), run as a role of the web process, no new service or domain. The
+ * listings stay in c0upons' own database; Redis only holds the queue and run
+ * records. Without REDIS_URL the package's local engine runs the same jobs
+ * in-process (dev, tests).
+ *
+ * Jobs:
+ *   peerpush  daily  peerpush.com/deals -> coupons (lib/peerpush-sync.ts)
+ *
+ * The older sources (r/couponcodes, Flipp) still run on their own pollers.
+ *
+ * The package's watchdog exits the process when Redis stops answering, which
+ * here would take the website down with it; instead the daemon is stopped and
+ * rebuilt on a fresh connection a few minutes later.
+ */
+
+const KEY = Symbol.for('c0upons.scraperDaemon');
+type Holder = { daemon?: Daemon; retry?: ReturnType<typeof setTimeout> };
+const holder = (): Holder => {
+  const g = globalThis as unknown as Record<symbol, Holder | undefined>;
+  return (g[KEY] ??= {});
+};
+
+const RETRY_MS = 5 * 60_000;
+
+function build(): Daemon {
+  const daemon = createDaemon({
+    name: 'c0upons',
+    redis: process.env.REDIS_URL || null,
+    concurrency: 2,
+    fetch: { userAgent: USER_AGENT, delayMs: 2000 },
+    onRedisDown: () => {
+      console.error('[daemon] redis unreachable: stopping, retrying in 5 min');
+      restartLater();
+    },
+    onStall: ({ job }: { job: string }) => console.error(`[daemon] ${job} has not succeeded within its staleAfter`),
+  });
+
+  daemon.job({
+    name: 'peerpush',
+    every: '1d',
+    staleAfter: '3d',
+    timeout: '5m',
+    run: async (ctx) => {
+      const r = await syncPeerpushDeals(getDb(), {
+        fetchText: async (url) => (await ctx.fetch.text(url, { accept: 'text/html', signal: ctx.signal })).text,
+        // The daemon schedules itself; this only stops a restart on the local engine re-reading at once.
+        minMinutesBetweenRuns: 20 * 60,
+      });
+      if (!r.skipped) {
+        console.log(`[peerpush] ${r.deals} deals, ${r.inserted} new, ${r.updated} updated, ${r.pruned} expired removed`);
+      }
+      const { added, ...counts } = r;
+      return { ...counts, added: added.slice(0, 20) };
+    },
+  });
+  return daemon;
+}
+
+function restartLater(): void {
+  const h = holder();
+  const old = h.daemon;
+  h.daemon = undefined;
+  old?.stop().catch(() => {});
+  if (h.retry) return;
+  h.retry = setTimeout(() => {
+    h.retry = undefined;
+    startDaemon();
+  }, RETRY_MS);
+  h.retry.unref?.();
+}
+
+/** Start once per process. `SCRAPER_DAEMON=0` turns it off. */
+export function startDaemon(): void {
+  if (process.env.SCRAPER_DAEMON === '0') return;
+  const h = holder();
+  if (h.daemon) return;
+  try {
+    h.daemon = build();
+    Promise.resolve(h.daemon.start()).catch((err: unknown) => {
+      console.error('[daemon] start failed:', err instanceof Error ? err.message : err);
+      restartLater();
+    });
+  } catch (err) {
+    console.error('[daemon] start failed:', err instanceof Error ? err.message : err);
+  }
+}
+
+export function getDaemon(): Daemon | undefined {
+  return holder().daemon;
+}

apps/web/lib/mcp-tools.ts

@@ -6,6 +6,7 @@ import type { McpTool, McpServerInfo } from './mcp-protocol';
 import { syncNichedbDeals } from './nichedb-sync';
 import { syncRedditCouponcodes } from './reddit-sync';
 import { syncFlippWeeklyAds } from './flipp-sync';
+import { syncPeerpushDeals } from './peerpush-sync';
 import { RECHECK_HOURS, ensureRevealColumns, revealDeps, revealForCoupon, sweepReveals } from './reveal-coupon';
 
 /**
@@ -23,6 +24,7 @@ export const SERVER: McpServerInfo = {
     'get_coupon for one, and reveal_code when a coupon has no code: a real browser reads its deal page and clicks what a ' +
     'shopper would (up to a minute). sync_deals pulls new deals in, sync_reddit reads the newest r/couponcodes posts, ' +
     'sync_grocery reads the next grocery weekly ads (Raley\'s, Safeway, Walmart and the rest), and ' +
+    'sync_peerpush reads the PeerPush founders\' deals board, and ' +
     'reveal_pending reads a few code-less pages.',
 };
 
@@ -150,6 +152,15 @@ export const TOOLS: McpTool[] = [
       return syncFlippWeeklyAds(getDb());
     },
   },
+  {
+    name: 'sync_peerpush',
+    description:
+      "Read peerpush.com/deals (discount codes software founders attach to their PeerPush listings) into stores and coupons, and remove deals past their valid-until date. The site's scraper daemon runs this daily; throttled to one run per hour.",
+    inputSchema: { type: 'object', properties: {}, additionalProperties: false },
+    async run() {
+      return syncPeerpushDeals(getDb());
+    },
+  },
   {
     name: 'reveal_pending',
     description: 'Read the deal pages of up to three coupons that have no code yet, most voted first, and say how many are still waiting.',