sh1pt-actions-fleet · Nov 2, 2026 · Nov 2, 2026 · Nov 2, 2026
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,6 @@
+**/node_modules
+.git
+.claude
+data
+*.db
+*.sqlite
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -8,12 +8,20 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
+    # The crawler's frontier and queues are tested against a real Redis.
+    services:
+      redis:
+        image: redis:8-alpine
+        ports: ['6379:6379']
+    env:
+      CRAWL_TEST_REDIS: redis://127.0.0.1:6379/15
     steps:
       - uses: actions/checkout@v4
       - uses: oven-sh/setup-bun@v2
       - run: bun install --frozen-lockfile
       - run: bun test
       - run: bunx tsc --noEmit
+      - run: bunx tsc --noEmit -p services/crawler
 
   # A private search engine is held to the strictest gate: nothing at medium or
   # above, dependencies included.

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -31,6 +31,7 @@ jobs:
             *-----BEGINOPENSSHPRIVATEKEY-----*)
               body=${flat#*-----BEGINOPENSSHPRIVATEKEY-----}
               body=${body%%-----ENDOPENSSHPRIVATEKEY-----*}
+              # threatcrush-disable-next-line secret-private-key  rebuilds the PIT_DEPLOY_KEY secret's armour at run time; no key material is committed
               { echo '-----BEGIN OPENSSH PRIVATE KEY-----'; printf '%s\n' "$body" | fold -w 70; echo '-----END OPENSSH PRIVATE KEY-----'; } > ~/.ssh/id_ed25519 ;;
             *) printf '%s' "$flat" | base64 -d > ~/.ssh/id_ed25519 ;;
           esac

diff --git a/bun.lock b/bun.lock
diff --git a/deploy/crawler/docker-compose.yml b/deploy/crawler/docker-compose.yml
@@ -0,0 +1,56 @@
+# pitbot on one box: the crawler, pit.world's own Redis (never a shared one),
+# and Obscura browser servers for JavaScript pages. Nothing listens beyond
+# loopback; the crawler cannot reach the host's own services (bridge network,
+# and every address is checked against private ranges before a request).
+#
+#   cd ~/www/pit-crawler && docker compose -f app/deploy/crawler/docker-compose.yml --env-file crawler.env -p pit-crawler up -d --build
+#
+# Health: curl 127.0.0.1:8807/healthz on the host.
+# Scale: `--scale crawler=N` on this box, or the same image on another box with
+# CRAWL_REDIS_URL pointed here. More JavaScript rendering: add obscura-N and
+# list it in OBSCURA_MCP_URLS.
+name: pit-crawler
+
+services:
+  redis:
+    image: redis:8-alpine
+    command: ['redis-server', '--appendonly', 'yes', '--maxmemory', '1gb', '--maxmemory-policy', 'noeviction']
+    volumes: ['redis:/data']
+    ports: ['127.0.0.1:6396:6379']
+    mem_limit: 1536m
+    restart: unless-stopped
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping']
+      interval: 15s
+
+  obscura-1: &obscura
+    image: h4ckf0r0day/obscura
+    command: ['mcp', '--http', '--host', '0.0.0.0', '--port', '3000', '--user-agent', 'Mozilla/5.0 (compatible; pitbot/0.1; +https://pit.world/bot)']
+    environment:
+      OBSCURA_MCP_TOKEN: ${OBSCURA_MCP_TOKEN:?set OBSCURA_MCP_TOKEN in crawler.env}
+    mem_limit: 1g
+    restart: unless-stopped
+
+  obscura-2: *obscura
+
+  crawler:
+    build:
+      context: ../..
+      dockerfile: services/crawler/Dockerfile
+    image: pit-crawler:latest
+    env_file: ${CRAWLER_ENV:-../../../crawler.env}
+    environment:
+      CRAWL_REDIS_URL: redis://redis:6379
+      CRAWL_HEALTH_HOST: 0.0.0.0
+      OBSCURA_MCP_URLS: http://obscura-1:3000/mcp,http://obscura-2:3000/mcp
+    depends_on:
+      redis: { condition: service_healthy }
+      obscura-1: { condition: service_started }
+      obscura-2: { condition: service_started }
+    ports: ['127.0.0.1:8807:8797']
+    mem_limit: 1536m
+    restart: unless-stopped
+    stop_grace_period: 30s
+
+volumes:
+  redis:
diff --git a/deploy/crawler/poll.sh b/deploy/crawler/poll.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Deploy-on-merge for pitbot: when main moves, rebuild and restart the crawler.
+# Runs from cron as the box's user (no root needed), e.g. every 5 minutes:
+#
+#   */5 * * * * ~/www/pit-crawler/app/deploy/crawler/poll.sh >> ~/www/pit-crawler/poll.log 2>&1
+#
+# Layout: ~/www/pit-crawler/{app (a checkout of moshcoder/pit.world), crawler.env (0600)}.
+set -euo pipefail
+
+ROOT=${PIT_CRAWLER_ROOT:-$HOME/www/pit-crawler}
+APP=$ROOT/app
+cd "$APP"
+
+git fetch -q origin main
+want=$(git rev-parse origin/main)
+have=$(git rev-parse HEAD)
+if [ "$want" = "$have" ] && [ "${1:-}" != "--force" ]; then exit 0; fi
+
+echo "$(date -u +%FT%TZ) deploying ${want:0:12} (was ${have:0:12})"
+git reset -q --hard "$want"
+docker compose -f deploy/crawler/docker-compose.yml --env-file "$ROOT/crawler.env" -p pit-crawler up -d --build --remove-orphans
+
+# Up and answering, or say so loudly.
+for _ in $(seq 1 20); do
+  if curl -fsS http://127.0.0.1:8807/healthz >/dev/null 2>&1; then
+    echo "$(date -u +%FT%TZ) ${want:0:12} healthy"
+    exit 0
+  fi
+  sleep 3
+done
+echo "$(date -u +%FT%TZ) ${want:0:12} NOT healthy after 60 s" >&2
+docker compose -p pit-crawler logs --tail 40 crawler >&2 || true
+exit 1
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "pitworld",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "mcpName": "world.pit/pit",
   "description": "pit.world MCP server: private web search and Moshpit name lookups for AI agents (stdio, no API key). Also a web app, API, CLI and TUI.",
   "license": "MIT",
@@ -20,14 +20,18 @@
     "README.md",
     "LICENSE"
   ],
+  "workspaces": [
+    "services/*"
+  ],
   "engines": {
     "bun": ">=1.1"
   },
   "scripts": {
     "start": "bun src/server.ts",
     "dev": "bun --watch src/server.ts",
     "test": "bun test",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit",
+    "crawler": "bun services/crawler/src/cli.ts"
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.32.2",

diff --git a/public/style.css b/public/style.css
@@ -269,3 +269,33 @@ form .btn { border: 0; cursor: pointer; }
 .business-previews{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:16px;margin:26px 0}.business-preview{border:1px solid var(--line);border-radius:10px;padding:20px;min-width:0}.business-preview.sponsored{border-color:var(--accent)}.business-preview h3{font-size:20px;line-height:1.3;margin:12px 0}.business-preview>p{font-size:14px;overflow-wrap:anywhere}.business-preview .small{font-size:11px;color:var(--muted)}.business-preview>a{font-size:13px}.business-offer{display:flex;justify-content:space-between;gap:20px;flex-wrap:wrap;margin:24px 0 0;border-top:1px solid var(--line);padding-top:24px;font-size:13px}.business-offer>span{color:var(--muted)}.paid-label{font-size:12px;color:var(--link);font-weight:700}.check{display:flex!important;align-items:start;gap:10px}.check>input{width:auto;flex-shrink:0}.listing-edit .row{flex-wrap:wrap}.listing-edit button{white-space:normal}
 @media(max-width:700px){.home-businesses{padding:22px;width:calc(100% - 28px)}.business-heading{align-items:start;flex-direction:column;gap:16px}.business-search,.business-previews{grid-template-columns:1fr}.business-search .btn{width:100%}.business-heading>a{white-space:normal}.business-offer{flex-direction:column;gap:12px}}
 .business-search input,.listing-edit input:not([type=checkbox]){min-height:46px;border:1px solid var(--line);border-radius:8px;background:var(--bg);color:var(--text);padding:11px 14px;font:16px var(--sans)}.listing-edit{max-width:680px;padding:24px;border:1px solid var(--line);border-radius:12px;background:var(--panel)}.listing-edit>label:not(.check){display:block;font-weight:600;margin:18px 0 8px}.listing-edit input:not([type=checkbox]){display:block;width:100%;max-width:100%;box-sizing:border-box}.listing-edit .check{margin:22px 0;line-height:1.6}.listing-edit .check input{margin-top:6px;accent-color:var(--accent)}.listing-edit .row{display:flex;gap:8px}.listing-edit h2{margin-top:32px}.business-search input:focus,.listing-edit input:focus{outline:2px solid var(--accent);outline-offset:2px}
+
+/* /status: pitbot's live crawler status */
+.crawl-status { max-width: 980px; }
+.crawl-health { padding: 12px 16px; border: 1px solid var(--line); border-left: 4px solid var(--muted); border-radius: 8px; background: var(--panel); }
+.crawl-health-healthy { border-left-color: var(--accent); }
+.crawl-health-idle { border-left-color: var(--muted); }
+.crawl-health-degraded { border-left-color: #f5b83d; }
+.crawl-health-stalled, .crawl-health-offline { border-left-color: var(--danger); }
+.stat-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(150px, 1fr)); gap: 10px; margin: 18px 0; }
+.stat { display: flex; flex-direction: column; gap: 2px; padding: 12px; border: 1px solid var(--line); border-radius: 8px; background: var(--panel); min-width: 0; }
+.stat b { font-size: 24px; font-variant-numeric: tabular-nums; }
+.stat-label, .stat small { color: var(--muted); font-size: 13px; }
+.crawl-chart { margin: 8px 0 0; }
+.crawl-chart svg { width: 100%; height: 90px; display: block; }
+.crawl-chart .bar-ok { fill: var(--accent); }
+.crawl-chart .bar-bad { fill: var(--danger); }
+.crawl-chart figcaption { color: var(--muted); font-size: 13px; }
+.crawl-jobs { width: 100%; border-collapse: collapse; font-size: 14px; display: block; overflow-x: auto; }
+.crawl-jobs th, .crawl-jobs td { text-align: left; padding: 8px; border-bottom: 1px solid var(--line); vertical-align: top; font-variant-numeric: tabular-nums; }
+.crawl-jobs small { color: var(--muted); }
+.state { padding: 1px 8px; border-radius: 999px; border: 1px solid var(--line); font-size: 12px; }
+.state-working { color: var(--accent); border-color: var(--accent); }
+.state-stalled { color: var(--danger); border-color: var(--danger); }
+.crawl-log { list-style: none; padding: 0; margin: 0; font-family: var(--mono); font-size: 13px; }
+.crawl-log li { padding: 4px 0; border-bottom: 1px solid var(--line); overflow-wrap: anywhere; }
+.crawl-log time, .crawl-log small, .crawl-log .site { color: var(--muted); }
+.crawl-log .kind { display: inline-block; min-width: 4.5em; color: var(--accent); }
+.crawl-log .ev-fail .kind { color: var(--danger); }
+.crawl-log .ev-skip .kind { color: var(--muted); }
+.crawl-log .rec { font-size: 12px; }
diff --git a/services/crawler/Dockerfile b/services/crawler/Dockerfile
@@ -0,0 +1,17 @@
+# pitbot, pit.world's crawler. Built from the repository root:
+#   docker build -f services/crawler/Dockerfile -t pit-crawler .
+FROM oven/bun:1.4-slim
+WORKDIR /app
+COPY package.json bun.lock ./
+COPY services/crawler/package.json services/crawler/package.json
+RUN bun install --frozen-lockfile --production
+COPY src ./src
+COPY services/crawler ./services/crawler
+# Dev boxes check out with umask 007; the runtime user must still read the code.
+RUN chmod -R a+rX /app
+ENV NODE_ENV=production CRAWL_HEALTH_PORT=8797
+USER bun
+# A blocked event loop or a dead Redis shows as unhealthy (watchdog-autoheal restarts it).
+HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
+  CMD bun -e "fetch('http://127.0.0.1:8797/healthz').then(r => process.exit(r.ok ? 0 : 1), () => process.exit(1))"
+CMD ["bun", "services/crawler/src/cli.ts", "worker"]
diff --git a/services/crawler/README.md b/services/crawler/README.md
@@ -0,0 +1,79 @@
+# pitbot: pit.world's crawler
+
+pit.world's own index. A BullMQ spider that reads up to 1,000 pages per site, renders
+JavaScript-only pages in [Obscura](https://github.com/h4ckf0r0day/obscura), enriches each page
+and publishes it as an [OpenSite](https://logicsrc.com/opensite) record to
+[nichedb.dev/c/sites](https://nichedb.dev/c/sites). pit.world then searches those records as one
+more engine (`pit` in `src/search.ts`), fused with the others.
+
+```
+seed (nichedb sites, ring, file, names)
+  -> pit-crawl-site    robots.txt, sitemaps, home page          (one job per site per cycle)
+  -> pit-crawl-page    fetch, extract, follow same-site links    (shallow first, polite per site)
+  -> pit-crawl-render  empty JS shells, read again in Obscura    (one page per Obscura server)
+  -> Redis outbox      { record, enrichment }
+  -> pit-crawl-publish POST nichedb.dev/api/v1/sites/records     (200 a call, every 5 s)
+```
+
+## What it records
+
+- **OpenSite record**: canonical (same-site only), kind, title, description, image, language,
+  author, feeds, published/modified dates, og/twitter tags verbatim, JSON-LD.
+- **Enrichment**: language, top keywords (become `topic:` tags on nichedb, so they are
+  searchable), a 1,000-character excerpt of the main text (the summary when the page has no
+  description), word and link counts, whether it was rendered, link depth, and a hash of the
+  text: an unchanged page is not written again.
+
+## Manners
+
+User agent `Mozilla/5.0 (compatible; pitbot/0.1; +https://pit.world/bot)`. robots.txt (a
+`pitbot` group, else `*`; longest match; Crawl-delay up to 30 s), `noindex`/`nofollow`, no
+`rel=nofollow|sponsored|ugc` links, one request per site at a time with a gap (1.5 s default),
+no forms/carts/logins, media and documents skipped, never a private address (checked by name
+and by DNS at every redirect hop).
+
+## Scale
+
+All state is in Redis (`pitcrawl:*` keys), so any number of `pit-crawler worker` processes on
+any number of boxes share one crawl. Admission and the 1,000-page cap are one Lua call (exact
+under races). A site inside its request gap moves its job back to delayed instead of holding a
+worker. `CRAWL_ROLES=page` runs fetch-only workers; `CRAWL_ROLES=render` next to Obscura.
+
+## Commands
+
+```
+bun services/crawler/src/cli.ts worker
+bun services/crawler/src/cli.ts seed example.com other.example
+bun services/crawler/src/cli.ts seed --from nichedb      # backfill: every site nichedb keeps
+bun services/crawler/src/cli.ts seed --from ring
+bun services/crawler/src/cli.ts stats
+bun services/crawler/src/cli.ts read https://example.com/ [--render]
+```
+
+## Settings
+
+| Env | Default | |
+| --- | --- | --- |
+| `CRAWL_REDIS_URL` | `redis://127.0.0.1:6396` | pit.world's own Redis, never a shared one |
+| `CRAWL_MAX_PAGES` | 1000 | pages per site per cycle |
+| `CRAWL_MAX_DEPTH` | 6 | link hops |
+| `CRAWL_CONCURRENCY` | 16 | pages at once per process |
+| `CRAWL_HOST_DELAY_MS` | 1500 | gap between two requests to one site |
+| `CRAWL_RECRAWL_DAYS` | 30 | a site is read again this often |
+| `CRAWL_DISCOVER` | 0 | 1: crawl linked sites too, up to `CRAWL_MAX_SITES` (100,000) |
+| `OBSCURA_MCP_URLS` | none | comma-separated `obscura mcp --http` endpoints |
+| `OBSCURA_MCP_TOKEN` | none | Obscura's bearer token (32+ bytes) |
+| `NICHEDB_URL` / `NICHEDB_API_KEY` | nichedb.dev / none | an admin key; without one records wait in the outbox |
+| `CRAWL_HEALTH_PORT` | 8797 | `/healthz` with queue depths and counters |
+
+## Deploy (dev2)
+
+`deploy/crawler/docker-compose.yml`: the crawler, its Redis, two Obscura servers, all on
+loopback (`/healthz` at 127.0.0.1:8807). `~/www/pit-crawler/{app,crawler.env}`;
+`deploy/crawler/poll.sh` from cron rebuilds on every new commit to main.
+
+## Tests
+
+`bun test` runs everything with fakes. `CRAWL_TEST_REDIS=redis://127.0.0.1:6399/15` adds the
+real-Redis frontier and queue tests (CI runs them), `OBSCURA_TEST_URL` + `OBSCURA_TEST_TOKEN` a
+real render.
diff --git a/services/crawler/package.json b/services/crawler/package.json
@@ -0,0 +1,20 @@
+{
+  "name": "@pitworld/crawler",
+  "version": "0.1.0",
+  "private": true,
+  "description": "pit.world's own index: a BullMQ spider that reads up to 1,000 pages per site (JavaScript pages through Obscura), enriches them and publishes OpenSite records to nichedb.dev/c/sites.",
+  "type": "module",
+  "bin": {
+    "pit-crawler": "src/cli.ts"
+  },
+  "scripts": {
+    "start": "bun src/cli.ts worker",
+    "test": "bun test",
+    "typecheck": "tsc --noEmit -p ."
+  },
+  "dependencies": {
+    "bullmq": "5.81.5",
+    "ioredis": "^5.8.2",
+    "node-html-parser": "^9.0.6"
+  }
+}