diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..549e4ed --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +**/node_modules +.git +.claude +data +*.db +*.sqlite diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c2fa95..e2dd336 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,12 +8,20 @@ on: jobs: test: runs-on: ubuntu-latest + # The crawler's frontier and queues are tested against a real Redis. + services: + redis: + image: redis:8-alpine + ports: ['6379:6379'] + env: + CRAWL_TEST_REDIS: redis://127.0.0.1:6379/15 steps: - uses: actions/checkout@v4 - uses: oven-sh/setup-bun@v2 - run: bun install --frozen-lockfile - run: bun test - run: bunx tsc --noEmit + - run: bunx tsc --noEmit -p services/crawler # A private search engine is held to the strictest gate: nothing at medium or # above, dependencies included. diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d4b02ab..9ef84d8 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -31,6 +31,7 @@ jobs: *-----BEGINOPENSSHPRIVATEKEY-----*) body=${flat#*-----BEGINOPENSSHPRIVATEKEY-----} body=${body%%-----ENDOPENSSHPRIVATEKEY-----*} + # threatcrush-disable-next-line secret-private-key rebuilds the PIT_DEPLOY_KEY secret's armour at run time; no key material is committed { echo '-----BEGIN OPENSSH PRIVATE KEY-----'; printf '%s\n' "$body" | fold -w 70; echo '-----END OPENSSH PRIVATE KEY-----'; } > ~/.ssh/id_ed25519 ;; *) printf '%s' "$flat" | base64 -d > ~/.ssh/id_ed25519 ;; esac diff --git a/bun.lock b/bun.lock index 4caf73f..c0916ea 100644 --- a/bun.lock +++ b/bun.lock @@ -22,12 +22,40 @@ "typescript": "^7.0.2", }, }, + "services/crawler": { + "name": "@pitworld/crawler", + "version": "0.1.0", + "bin": { + "pit-crawler": "src/cli.ts", + }, + "dependencies": { + "bullmq": "5.81.5", + "ioredis": "^5.8.2", + "node-html-parser": "^9.0.6", + }, + }, }, "packages": { "@hono/node-server": ["@hono/node-server@2.1.5", "", { "peerDependencies": { "hono": "^4 || ^5.0.0-0" } }, "sha512-MgP/I06GdvsilDspMk/qmRzpk8LOUStP6jFAbSRi40W09DhoyUn+Mj7eyKICPqeYKybOjVFLVIAQzvu1bfIa7Q=="], + "@ioredis/commands": ["@ioredis/commands@1.10.0", "", {}, "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q=="], + "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.32.2", "", { "dependencies": { "@hono/node-server": "^1.19.9 || ^2.0.5", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-SWsVtH/0vULSQStoZ6xxH2K09Li8fXYTMC64ZeogCdpZ3XSi1eCPOByV1OpuJxuI3OcmHhosDWYyii07tpPsLw=="], + "@msgpackr-extract/msgpackr-extract-darwin-arm64": ["@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.6", "", { "os": "darwin", "cpu": "arm64" }, "sha512-ugMD1XQPQ0usC3F1frfZzZsNs9uf9Xyxe86GO078Mg0dVY+FQA8TlPL8P5EQwavZs0dUe2LjWJnt4TyQl3pUvw=="], + + "@msgpackr-extract/msgpackr-extract-darwin-x64": ["@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.6", "", { "os": "darwin", "cpu": "x64" }, "sha512-9ESac1RNLXH7jPZri+aOSoWOG3YpxFUdxgcrjlcANf2wDbEUf55bY3R+uGQqYkqcFLF91EkoAcDsJPbbGj84Sw=="], + + "@msgpackr-extract/msgpackr-extract-linux-arm": ["@msgpackr-extract/msgpackr-extract-linux-arm@3.0.6", "", { "os": "linux", "cpu": "arm" }, "sha512-tAObEkjmQ77IgZfg6UEyEUgKP9vLVMkq2KK8WNa5P7hIieevSMHxyKcqM4zCV+TiZu8q7XmWLrqqlSuaZ5Z0zA=="], + + "@msgpackr-extract/msgpackr-extract-linux-arm64": ["@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.6", "", { "os": "linux", "cpu": "arm64" }, "sha512-zHceGhbGw8RQG6oSKK/43ieOjutwb/yVXa5kmdJCfFCLbTdOkd4TGfvbiic70VJf8uomuV4AwOfFxe1i2/ZLmQ=="], + + "@msgpackr-extract/msgpackr-extract-linux-x64": ["@msgpackr-extract/msgpackr-extract-linux-x64@3.0.6", "", { "os": "linux", "cpu": "x64" }, "sha512-7zKNTbhIGdVO4gsGFNd39g5Go3jN+fuizvWe8SrbGSck9Pcr8x9I5GbtUX1JzBLMaUuI/tcB0JKEYn8RNjKhFg=="], + + "@msgpackr-extract/msgpackr-extract-win32-x64": ["@msgpackr-extract/msgpackr-extract-win32-x64@3.0.6", "", { "os": "win32", "cpu": "x64" }, "sha512-J1mT1tp4v9aqcRwUtFrdJ94yei2SlIKm243nWK5bCyOPNnnNknaPzjG6a2Aa5OSfXE36rrth0X2OPHpispZTJw=="], + + "@pitworld/crawler": ["@pitworld/crawler@workspace:services/crawler"], + "@profullstack/affiliates": ["@profullstack/affiliates@0.2.0", "", { "dependencies": { "@profullstack/referrals": "^0.1.0" }, "bin": { "affiliates": "dist/cli.js" } }, "sha512-fbP5uiS8Pa5g8mk5WFD79U4Cj7BUPH6Qy0tQP7ZXE9gfxHOHirHASmZ/rAk8MqnmzGOZTuwq9WwOHbmoFV2TSA=="], "@profullstack/bridges": ["@profullstack/bridges@0.2.0", "", { "bin": { "bridges": "bin/bridges.js" } }, "sha512-ai74Zl1mQN2WZDVb93qiwfpqjCDCg/TZkxgFHFn4Rb/Cxdgg2kUst9ep0B609WoulnNjsXN3IU4IMhpzmypg7w=="], @@ -100,6 +128,10 @@ "body-parser": ["body-parser@2.4.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^2.1.0", "debug": "^4.4.3", "http-errors": "^2.0.1", "iconv-lite": "^0.7.3", "on-finished": "^2.4.1", "qs": "^6.16.1", "raw-body": "^3.0.2", "type-is": "^2.1.0" } }, "sha512-Ifk1KcLrxaGt1occQagQNFNNOqRt1LFuehEnHHzB8Z6cRgIxRU1TBaiIsUiCdA+gy5PL7Y1IX31DNKkRmxtXjQ=="], + "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], + + "bullmq": ["bullmq@5.81.5", "", { "dependencies": { "cron-parser": "4.9.0", "ioredis": "5.11.1", "msgpackr": "2.0.5", "node-abort-controller": "3.1.1", "semver": "7.8.5", "tslib": "2.8.1" }, "peerDependencies": { "redis": ">=5.0.0" }, "optionalPeers": ["redis"] }, "sha512-pW7pFc/X+9AEIx91TWTdIpZ1icfbJnA0unsyzD7VICjCUw8a3qedSjej5dxxn6LjfM1kr2jvalOCGbCmoA5Kxg=="], + "bun-types": ["bun-types@1.4.4", "", { "dependencies": { "@types/node": "*", "undici-types": "*" } }, "sha512-yjrJh5oGX7mMPKGd19YXyIjmEuj4G4zNC30bs7PM9Az7NQ3ndF3j6hbeRdvJTRQEhJ/FYQwW0a2moweEi2hMxQ=="], "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="], @@ -108,6 +140,8 @@ "call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="], + "cluster-key-slot": ["cluster-key-slot@1.1.1", "", {}, "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw=="], + "content-disposition": ["content-disposition@2.0.1", "", {}, "sha512-e+H0ZXHSWYrENhQzw1LPuP4oF5MzVKmDU6d3hxlvaPEYLLg62MxtQNPRx4SYSuYJSBUgnQIG4HIN2tEtNv7Dog=="], "content-type": ["content-type@1.0.5", "", {}, "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA=="], @@ -118,18 +152,38 @@ "cors": ["cors@2.8.6", "", { "dependencies": { "object-assign": "^4", "vary": "^1" } }, "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw=="], + "cron-parser": ["cron-parser@4.9.0", "", { "dependencies": { "luxon": "^3.2.1" } }, "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q=="], + "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], + "css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="], + + "css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="], + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" }, "peerDependencies": { "supports-color": "*" }, "optionalPeers": ["supports-color"] }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + "denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="], + "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="], + "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], + + "dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="], + + "domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="], + + "domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="], + + "domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="], + "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="], "encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="], + "entities": ["entities@8.1.0", "", {}, "sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA=="], + "es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="], "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="], @@ -178,6 +232,8 @@ "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], + "ioredis": ["ioredis@5.11.1", "", { "dependencies": { "@ioredis/commands": "1.10.0", "cluster-key-slot": "1.1.1", "debug": "4.4.3", "denque": "2.1.0", "redis-errors": "1.2.0", "redis-parser": "3.0.0", "standard-as-callback": "2.1.0" } }, "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A=="], + "ip-address": ["ip-address@10.7.5", "", {}, "sha512-t2v0rkILYv9WXmSL7TpAaU3HXgaFYOPUcH3QHgwQ++gpJuH3bqyMy4Fvt/DJCTKwzH3Cf+bAp5W/mFL39KPLzA=="], "ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], @@ -192,6 +248,8 @@ "json-schema-typed": ["json-schema-typed@8.0.2", "", {}, "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA=="], + "luxon": ["luxon@3.7.2", "", {}, "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew=="], + "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="], "media-typer": ["media-typer@1.1.1", "", {}, "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ=="], @@ -204,8 +262,20 @@ "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "msgpackr": ["msgpackr@2.0.5", "", { "optionalDependencies": { "msgpackr-extract": "^3.0.4" } }, "sha512-cef05H/dSYpLpqp3sj/qyZh5vhUYCalnaLO7j1yOmpsR0y/XwLVtK7r5gn+U/F7CTEfMowcGhlUQJDLcLf7jcA=="], + + "msgpackr-extract": ["msgpackr-extract@3.0.6", "", { "dependencies": { "node-gyp-build-optional-packages": "5.2.2" }, "optionalDependencies": { "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.6", "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.6", "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.6", "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.6", "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.6", "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.6" }, "hasInstallScript": true, "bin": { "download-msgpackr-prebuilds": "bin/download-prebuilds.js" } }, "sha512-prUB8+Gc/1rEzaNo+3Rjo0Hdi6gFjQ9Td0mDsvAIkSGUyp6EADdEBiU71ZP5Mj4jCG+vai0qSNV+yisQAMYuSQ=="], + "negotiator": ["negotiator@1.1.0", "", { "dependencies": { "content-type": "^2.1.0" } }, "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg=="], + "node-abort-controller": ["node-abort-controller@3.1.1", "", {}, "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ=="], + + "node-gyp-build-optional-packages": ["node-gyp-build-optional-packages@5.2.2", "", { "dependencies": { "detect-libc": "^2.0.1" }, "bin": { "node-gyp-build-optional-packages": "bin.js", "node-gyp-build-optional-packages-optional": "optional.js", "node-gyp-build-optional-packages-test": "build-test.js" } }, "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw=="], + + "node-html-parser": ["node-html-parser@9.0.6", "", { "dependencies": { "css-select": "^5.1.0", "entities": "^8.0.0" } }, "sha512-q3ETJ2gU+VlERyYbiJ5JQ6+mMVV1Bc6mZQ7MdJ30moq+h3oN0kQGj3rWDORaSX1SOLSkPbY/bN+sBzDlliVOBA=="], + + "nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="], + "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="], @@ -230,12 +300,18 @@ "raw-body": ["raw-body@3.0.2", "", { "dependencies": { "bytes": "~3.1.2", "http-errors": "~2.0.1", "iconv-lite": "~0.7.0", "unpipe": "~1.0.0" } }, "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA=="], + "redis-errors": ["redis-errors@1.2.0", "", {}, "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w=="], + + "redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="], + "require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="], "router": ["router@2.2.0", "", { "dependencies": { "debug": "^4.4.0", "depd": "^2.0.0", "is-promise": "^4.0.0", "parseurl": "^1.3.3", "path-to-regexp": "^8.0.0" } }, "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ=="], "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], + "semver": ["semver@7.8.5", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA=="], + "send": ["send@1.2.1", "", { "dependencies": { "debug": "^4.4.3", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", "http-errors": "^2.0.1", "mime-types": "^3.0.2", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", "statuses": "^2.0.2" } }, "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ=="], "serve-static": ["serve-static@2.2.1", "", { "dependencies": { "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "parseurl": "^1.3.3", "send": "^1.2.0" } }, "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw=="], @@ -254,10 +330,14 @@ "side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="], + "standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="], + "statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="], "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + "type-is": ["type-is@2.1.0", "", { "dependencies": { "content-type": "^2.0.0", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA=="], "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], @@ -278,6 +358,8 @@ "body-parser/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], + "dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], + "express/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], "negotiator/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], diff --git a/deploy/crawler/docker-compose.yml b/deploy/crawler/docker-compose.yml new file mode 100644 index 0000000..ef021ea --- /dev/null +++ b/deploy/crawler/docker-compose.yml @@ -0,0 +1,56 @@ +# pitbot on one box: the crawler, pit.world's own Redis (never a shared one), +# and Obscura browser servers for JavaScript pages. Nothing listens beyond +# loopback; the crawler cannot reach the host's own services (bridge network, +# and every address is checked against private ranges before a request). +# +# cd ~/www/pit-crawler && docker compose -f app/deploy/crawler/docker-compose.yml --env-file crawler.env -p pit-crawler up -d --build +# +# Health: curl 127.0.0.1:8807/healthz on the host. +# Scale: `--scale crawler=N` on this box, or the same image on another box with +# CRAWL_REDIS_URL pointed here. More JavaScript rendering: add obscura-N and +# list it in OBSCURA_MCP_URLS. +name: pit-crawler + +services: + redis: + image: redis:8-alpine + command: ['redis-server', '--appendonly', 'yes', '--maxmemory', '1gb', '--maxmemory-policy', 'noeviction'] + volumes: ['redis:/data'] + ports: ['127.0.0.1:6396:6379'] + mem_limit: 1536m + restart: unless-stopped + healthcheck: + test: ['CMD', 'redis-cli', 'ping'] + interval: 15s + + obscura-1: &obscura + image: h4ckf0r0day/obscura + command: ['mcp', '--http', '--host', '0.0.0.0', '--port', '3000', '--user-agent', 'Mozilla/5.0 (compatible; pitbot/0.1; +https://pit.world/bot)'] + environment: + OBSCURA_MCP_TOKEN: ${OBSCURA_MCP_TOKEN:?set OBSCURA_MCP_TOKEN in crawler.env} + mem_limit: 1g + restart: unless-stopped + + obscura-2: *obscura + + crawler: + build: + context: ../.. + dockerfile: services/crawler/Dockerfile + image: pit-crawler:latest + env_file: ${CRAWLER_ENV:-../../../crawler.env} + environment: + CRAWL_REDIS_URL: redis://redis:6379 + CRAWL_HEALTH_HOST: 0.0.0.0 + OBSCURA_MCP_URLS: http://obscura-1:3000/mcp,http://obscura-2:3000/mcp + depends_on: + redis: { condition: service_healthy } + obscura-1: { condition: service_started } + obscura-2: { condition: service_started } + ports: ['127.0.0.1:8807:8797'] + mem_limit: 1536m + restart: unless-stopped + stop_grace_period: 30s + +volumes: + redis: diff --git a/deploy/crawler/poll.sh b/deploy/crawler/poll.sh new file mode 100644 index 0000000..089b59a --- /dev/null +++ b/deploy/crawler/poll.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Deploy-on-merge for pitbot: when main moves, rebuild and restart the crawler. +# Runs from cron as the box's user (no root needed), e.g. every 5 minutes: +# +# */5 * * * * ~/www/pit-crawler/app/deploy/crawler/poll.sh >> ~/www/pit-crawler/poll.log 2>&1 +# +# Layout: ~/www/pit-crawler/{app (a checkout of moshcoder/pit.world), crawler.env (0600)}. +set -euo pipefail + +ROOT=${PIT_CRAWLER_ROOT:-$HOME/www/pit-crawler} +APP=$ROOT/app +cd "$APP" + +git fetch -q origin main +want=$(git rev-parse origin/main) +have=$(git rev-parse HEAD) +if [ "$want" = "$have" ] && [ "${1:-}" != "--force" ]; then exit 0; fi + +echo "$(date -u +%FT%TZ) deploying ${want:0:12} (was ${have:0:12})" +git reset -q --hard "$want" +docker compose -f deploy/crawler/docker-compose.yml --env-file "$ROOT/crawler.env" -p pit-crawler up -d --build --remove-orphans + +# Up and answering, or say so loudly. +for _ in $(seq 1 20); do + if curl -fsS http://127.0.0.1:8807/healthz >/dev/null 2>&1; then + echo "$(date -u +%FT%TZ) ${want:0:12} healthy" + exit 0 + fi + sleep 3 +done +echo "$(date -u +%FT%TZ) ${want:0:12} NOT healthy after 60 s" >&2 +docker compose -p pit-crawler logs --tail 40 crawler >&2 || true +exit 1 diff --git a/package.json b/package.json index 7bf390f..ff63374 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pitworld", - "version": "0.12.0", + "version": "0.13.0", "mcpName": "world.pit/pit", "description": "pit.world MCP server: private web search and Moshpit name lookups for AI agents (stdio, no API key). Also a web app, API, CLI and TUI.", "license": "MIT", @@ -20,6 +20,9 @@ "README.md", "LICENSE" ], + "workspaces": [ + "services/*" + ], "engines": { "bun": ">=1.1" }, @@ -27,7 +30,8 @@ "start": "bun src/server.ts", "dev": "bun --watch src/server.ts", "test": "bun test", - "typecheck": "tsc --noEmit" + "typecheck": "tsc --noEmit", + "crawler": "bun services/crawler/src/cli.ts" }, "dependencies": { "@modelcontextprotocol/sdk": "^1.32.2", diff --git a/public/style.css b/public/style.css index 7a9e309..eaeeaa5 100644 --- a/public/style.css +++ b/public/style.css @@ -269,3 +269,33 @@ form .btn { border: 0; cursor: pointer; } .business-previews{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:16px;margin:26px 0}.business-preview{border:1px solid var(--line);border-radius:10px;padding:20px;min-width:0}.business-preview.sponsored{border-color:var(--accent)}.business-preview h3{font-size:20px;line-height:1.3;margin:12px 0}.business-preview>p{font-size:14px;overflow-wrap:anywhere}.business-preview .small{font-size:11px;color:var(--muted)}.business-preview>a{font-size:13px}.business-offer{display:flex;justify-content:space-between;gap:20px;flex-wrap:wrap;margin:24px 0 0;border-top:1px solid var(--line);padding-top:24px;font-size:13px}.business-offer>span{color:var(--muted)}.paid-label{font-size:12px;color:var(--link);font-weight:700}.check{display:flex!important;align-items:start;gap:10px}.check>input{width:auto;flex-shrink:0}.listing-edit .row{flex-wrap:wrap}.listing-edit button{white-space:normal} @media(max-width:700px){.home-businesses{padding:22px;width:calc(100% - 28px)}.business-heading{align-items:start;flex-direction:column;gap:16px}.business-search,.business-previews{grid-template-columns:1fr}.business-search .btn{width:100%}.business-heading>a{white-space:normal}.business-offer{flex-direction:column;gap:12px}} .business-search input,.listing-edit input:not([type=checkbox]){min-height:46px;border:1px solid var(--line);border-radius:8px;background:var(--bg);color:var(--text);padding:11px 14px;font:16px var(--sans)}.listing-edit{max-width:680px;padding:24px;border:1px solid var(--line);border-radius:12px;background:var(--panel)}.listing-edit>label:not(.check){display:block;font-weight:600;margin:18px 0 8px}.listing-edit input:not([type=checkbox]){display:block;width:100%;max-width:100%;box-sizing:border-box}.listing-edit .check{margin:22px 0;line-height:1.6}.listing-edit .check input{margin-top:6px;accent-color:var(--accent)}.listing-edit .row{display:flex;gap:8px}.listing-edit h2{margin-top:32px}.business-search input:focus,.listing-edit input:focus{outline:2px solid var(--accent);outline-offset:2px} + +/* /status: pitbot's live crawler status */ +.crawl-status { max-width: 980px; } +.crawl-health { padding: 12px 16px; border: 1px solid var(--line); border-left: 4px solid var(--muted); border-radius: 8px; background: var(--panel); } +.crawl-health-healthy { border-left-color: var(--accent); } +.crawl-health-idle { border-left-color: var(--muted); } +.crawl-health-degraded { border-left-color: #f5b83d; } +.crawl-health-stalled, .crawl-health-offline { border-left-color: var(--danger); } +.stat-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(150px, 1fr)); gap: 10px; margin: 18px 0; } +.stat { display: flex; flex-direction: column; gap: 2px; padding: 12px; border: 1px solid var(--line); border-radius: 8px; background: var(--panel); min-width: 0; } +.stat b { font-size: 24px; font-variant-numeric: tabular-nums; } +.stat-label, .stat small { color: var(--muted); font-size: 13px; } +.crawl-chart { margin: 8px 0 0; } +.crawl-chart svg { width: 100%; height: 90px; display: block; } +.crawl-chart .bar-ok { fill: var(--accent); } +.crawl-chart .bar-bad { fill: var(--danger); } +.crawl-chart figcaption { color: var(--muted); font-size: 13px; } +.crawl-jobs { width: 100%; border-collapse: collapse; font-size: 14px; display: block; overflow-x: auto; } +.crawl-jobs th, .crawl-jobs td { text-align: left; padding: 8px; border-bottom: 1px solid var(--line); vertical-align: top; font-variant-numeric: tabular-nums; } +.crawl-jobs small { color: var(--muted); } +.state { padding: 1px 8px; border-radius: 999px; border: 1px solid var(--line); font-size: 12px; } +.state-working { color: var(--accent); border-color: var(--accent); } +.state-stalled { color: var(--danger); border-color: var(--danger); } +.crawl-log { list-style: none; padding: 0; margin: 0; font-family: var(--mono); font-size: 13px; } +.crawl-log li { padding: 4px 0; border-bottom: 1px solid var(--line); overflow-wrap: anywhere; } +.crawl-log time, .crawl-log small, .crawl-log .site { color: var(--muted); } +.crawl-log .kind { display: inline-block; min-width: 4.5em; color: var(--accent); } +.crawl-log .ev-fail .kind { color: var(--danger); } +.crawl-log .ev-skip .kind { color: var(--muted); } +.crawl-log .rec { font-size: 12px; } diff --git a/services/crawler/Dockerfile b/services/crawler/Dockerfile new file mode 100644 index 0000000..f4d0e1d --- /dev/null +++ b/services/crawler/Dockerfile @@ -0,0 +1,17 @@ +# pitbot, pit.world's crawler. Built from the repository root: +# docker build -f services/crawler/Dockerfile -t pit-crawler . +FROM oven/bun:1.4-slim +WORKDIR /app +COPY package.json bun.lock ./ +COPY services/crawler/package.json services/crawler/package.json +RUN bun install --frozen-lockfile --production +COPY src ./src +COPY services/crawler ./services/crawler +# Dev boxes check out with umask 007; the runtime user must still read the code. +RUN chmod -R a+rX /app +ENV NODE_ENV=production CRAWL_HEALTH_PORT=8797 +USER bun +# A blocked event loop or a dead Redis shows as unhealthy (watchdog-autoheal restarts it). +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ + CMD bun -e "fetch('http://127.0.0.1:8797/healthz').then(r => process.exit(r.ok ? 0 : 1), () => process.exit(1))" +CMD ["bun", "services/crawler/src/cli.ts", "worker"] diff --git a/services/crawler/README.md b/services/crawler/README.md new file mode 100644 index 0000000..51bbf93 --- /dev/null +++ b/services/crawler/README.md @@ -0,0 +1,79 @@ +# pitbot: pit.world's crawler + +pit.world's own index. A BullMQ spider that reads up to 1,000 pages per site, renders +JavaScript-only pages in [Obscura](https://github.com/h4ckf0r0day/obscura), enriches each page +and publishes it as an [OpenSite](https://logicsrc.com/opensite) record to +[nichedb.dev/c/sites](https://nichedb.dev/c/sites). pit.world then searches those records as one +more engine (`pit` in `src/search.ts`), fused with the others. + +``` +seed (nichedb sites, ring, file, names) + -> pit-crawl-site robots.txt, sitemaps, home page (one job per site per cycle) + -> pit-crawl-page fetch, extract, follow same-site links (shallow first, polite per site) + -> pit-crawl-render empty JS shells, read again in Obscura (one page per Obscura server) + -> Redis outbox { record, enrichment } + -> pit-crawl-publish POST nichedb.dev/api/v1/sites/records (200 a call, every 5 s) +``` + +## What it records + +- **OpenSite record**: canonical (same-site only), kind, title, description, image, language, + author, feeds, published/modified dates, og/twitter tags verbatim, JSON-LD. +- **Enrichment**: language, top keywords (become `topic:` tags on nichedb, so they are + searchable), a 1,000-character excerpt of the main text (the summary when the page has no + description), word and link counts, whether it was rendered, link depth, and a hash of the + text: an unchanged page is not written again. + +## Manners + +User agent `Mozilla/5.0 (compatible; pitbot/0.1; +https://pit.world/bot)`. robots.txt (a +`pitbot` group, else `*`; longest match; Crawl-delay up to 30 s), `noindex`/`nofollow`, no +`rel=nofollow|sponsored|ugc` links, one request per site at a time with a gap (1.5 s default), +no forms/carts/logins, media and documents skipped, never a private address (checked by name +and by DNS at every redirect hop). + +## Scale + +All state is in Redis (`pitcrawl:*` keys), so any number of `pit-crawler worker` processes on +any number of boxes share one crawl. Admission and the 1,000-page cap are one Lua call (exact +under races). A site inside its request gap moves its job back to delayed instead of holding a +worker. `CRAWL_ROLES=page` runs fetch-only workers; `CRAWL_ROLES=render` next to Obscura. + +## Commands + +``` +bun services/crawler/src/cli.ts worker +bun services/crawler/src/cli.ts seed example.com other.example +bun services/crawler/src/cli.ts seed --from nichedb # backfill: every site nichedb keeps +bun services/crawler/src/cli.ts seed --from ring +bun services/crawler/src/cli.ts stats +bun services/crawler/src/cli.ts read https://example.com/ [--render] +``` + +## Settings + +| Env | Default | | +| --- | --- | --- | +| `CRAWL_REDIS_URL` | `redis://127.0.0.1:6396` | pit.world's own Redis, never a shared one | +| `CRAWL_MAX_PAGES` | 1000 | pages per site per cycle | +| `CRAWL_MAX_DEPTH` | 6 | link hops | +| `CRAWL_CONCURRENCY` | 16 | pages at once per process | +| `CRAWL_HOST_DELAY_MS` | 1500 | gap between two requests to one site | +| `CRAWL_RECRAWL_DAYS` | 30 | a site is read again this often | +| `CRAWL_DISCOVER` | 0 | 1: crawl linked sites too, up to `CRAWL_MAX_SITES` (100,000) | +| `OBSCURA_MCP_URLS` | none | comma-separated `obscura mcp --http` endpoints | +| `OBSCURA_MCP_TOKEN` | none | Obscura's bearer token (32+ bytes) | +| `NICHEDB_URL` / `NICHEDB_API_KEY` | nichedb.dev / none | an admin key; without one records wait in the outbox | +| `CRAWL_HEALTH_PORT` | 8797 | `/healthz` with queue depths and counters | + +## Deploy (dev2) + +`deploy/crawler/docker-compose.yml`: the crawler, its Redis, two Obscura servers, all on +loopback (`/healthz` at 127.0.0.1:8807). `~/www/pit-crawler/{app,crawler.env}`; +`deploy/crawler/poll.sh` from cron rebuilds on every new commit to main. + +## Tests + +`bun test` runs everything with fakes. `CRAWL_TEST_REDIS=redis://127.0.0.1:6399/15` adds the +real-Redis frontier and queue tests (CI runs them), `OBSCURA_TEST_URL` + `OBSCURA_TEST_TOKEN` a +real render. diff --git a/services/crawler/package.json b/services/crawler/package.json new file mode 100644 index 0000000..4afc698 --- /dev/null +++ b/services/crawler/package.json @@ -0,0 +1,20 @@ +{ + "name": "@pitworld/crawler", + "version": "0.1.0", + "private": true, + "description": "pit.world's own index: a BullMQ spider that reads up to 1,000 pages per site (JavaScript pages through Obscura), enriches them and publishes OpenSite records to nichedb.dev/c/sites.", + "type": "module", + "bin": { + "pit-crawler": "src/cli.ts" + }, + "scripts": { + "start": "bun src/cli.ts worker", + "test": "bun test", + "typecheck": "tsc --noEmit -p ." + }, + "dependencies": { + "bullmq": "5.81.5", + "ioredis": "^5.8.2", + "node-html-parser": "^9.0.6" + } +} diff --git a/services/crawler/src/cli.ts b/services/crawler/src/cli.ts new file mode 100644 index 0000000..bb326c7 --- /dev/null +++ b/services/crawler/src/cli.ts @@ -0,0 +1,136 @@ +#!/usr/bin/env bun +/** + * pit-crawler: pit.world's spider. + * + * pit-crawler worker run the roles in CRAWL_ROLES (default all) until stopped + * pit-crawler seed crawl these sites + * pit-crawler seed --from nichedb|ring backfill: every site nichedb keeps, or the ring + * pit-crawler seed --file one site per line + * pit-crawler stats queue depths and counters, as JSON + * pit-crawler read [--render] read one page now and print its record (no Redis) + */ +import { readFileSync } from 'node:fs' +import { configFromEnv, VERSION } from './config.ts' +import { extract, needsRender } from './extract.ts' +import { politeGet } from './fetcher.ts' +import { ObscuraClient } from './obscura.ts' +import { createRuntime, report, snapshot, startWorkers } from './queues.ts' +import { sitesFromNichedb, sitesFromRing, sitesFromText } from './seed.ts' +import { normalizeUrl, resolvesPublic, siteOf } from './url.ts' + +const log = (line: string) => console.log(`${new Date().toISOString()} ${line}`) + +function flag(args: string[], name: string): string | null { + const i = args.indexOf(name) + return i >= 0 ? (args[i + 1] ?? '') : null +} + +async function worker() { + const config = configFromEnv() + const running = await startWorkers(config, log) + const { runtime } = running + + // Restart instead of hanging: a Redis that stops answering for a minute ends the process, systemd starts it again. + let misses = 0 + const watchdog = setInterval(async () => { + try { + await Promise.race([runtime.redis.ping(), new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), 5_000))]) + misses = 0 + } catch { + if (++misses >= 4) { + log('watchdog: Redis has not answered for a minute, exiting') + process.exit(1) + } + } + }, 15_000) + + const server = config.healthPort + ? Bun.serve({ + hostname: config.healthHost, + port: config.healthPort, + async fetch(request) { + const path = new URL(request.url).pathname + if (path !== '/healthz' && path !== '/stats') return new Response('not found', { status: 404 }) + try { + // /healthz is the cheap one (Docker asks every 30 s); /stats is what pit.world/status reads. + const body = path === '/stats' ? await report(runtime, config) : await snapshot(runtime) + return Response.json({ ok: true, version: VERSION, roles: config.roles, ...body }) + } catch (error) { + return Response.json({ ok: false, error: (error as Error).message }, { status: 503 }) + } + }, + }) + : null + + const stop = async () => { + log('stopping') + clearInterval(watchdog) + server?.stop() + await Promise.race([running.close(), new Promise((resolve) => setTimeout(resolve, 20_000))]) + process.exit(0) + } + process.on('SIGTERM', stop) + process.on('SIGINT', stop) +} + +async function seed(args: string[]) { + const config = configFromEnv() + const from = flag(args, '--from') + const file = flag(args, '--file') + const limit = Number(flag(args, '--limit')) || undefined + let sites: string[] = [] + if (from === 'nichedb') sites = await sitesFromNichedb(config.nichedbUrl, { ...(limit ? { limit } : {}) }) + else if (from === 'ring') sites = sitesFromRing() + else if (from) throw new Error(`unknown --from ${from} (nichedb or ring)`) + if (file) sites.push(...sitesFromText(readFileSync(file, 'utf8'))) + sites.push(...sitesFromText(args.filter((a, i) => !a.startsWith('--') && !args[i - 1]?.startsWith('--')).join(' '))) + if (limit) sites = sites.slice(0, limit) + const runtime = createRuntime(config, log) + try { + const out = await runtime.crawler.scheduleSites([...new Set(sites)], from ?? (file ? 'file' : 'cli')) + console.log(JSON.stringify({ found: sites.length, ...out })) + } finally { + await runtime.close() + } +} + +async function stats() { + const runtime = createRuntime(configFromEnv(), log) + try { + console.log(JSON.stringify(await snapshot(runtime), null, 2)) + } finally { + await runtime.close() + } +} + +async function read(args: string[]) { + const config = configFromEnv() + const url = normalizeUrl(args.find((a) => !a.startsWith('--')) ?? '') + if (!url) throw new Error('read ') + const r = await politeGet(url, { userAgent: config.userAgent, resolver: resolvesPublic }) + let doc = extract(r.body, normalizeUrl(r.finalUrl) ?? url, { crawler: config.userAgent }) + const shell = needsRender(r.body, doc.textLength) + if ((args.includes('--render') || shell) && config.obscuraUrls[0]) { + const rendered = await new ObscuraClient(config.obscuraUrls[0], config.obscuraToken).render(url) + doc = extract(rendered.html, normalizeUrl(rendered.url) ?? url, { rendered: true, crawler: config.userAgent }) + } + console.log( + JSON.stringify( + { status: r.status, shell, record: { ...doc.record, cards: undefined, jsonld: doc.record.jsonld.length }, enrichment: doc.enrichment, internal: doc.internal.length, external: doc.external.slice(0, 10), site: siteOf(url) }, + null, + 2, + ), + ) +} + +const [command, ...rest] = process.argv.slice(2) +const commands: Record Promise> = { worker, seed, stats: () => stats(), read } +const run = command ? commands[command] : undefined +if (!run) { + console.log('usage: pit-crawler worker | seed [--from nichedb|ring] [--file path] [--limit n] | stats | read [--render]') + process.exit(command ? 1 : 0) +} +run(rest).catch((error) => { + console.error(error instanceof Error ? error.message : error) + process.exit(1) +}) diff --git a/services/crawler/src/config.ts b/services/crawler/src/config.ts new file mode 100644 index 0000000..021af0c --- /dev/null +++ b/services/crawler/src/config.ts @@ -0,0 +1,81 @@ +/** + * Everything the crawler reads from its environment, in one place. + * + * The defaults are sized for one modest box (dev2 shares its RAM with a + * fleet): scale out by starting more `pit-crawler worker` processes against + * the same Redis, on this box or any other, not by raising one process's + * concurrency until it falls over. Render workers scale with Obscura + * servers, one page at a time each. + */ + +export const VERSION = '0.1.0' +export const BOT = 'pitbot' +export const USER_AGENT = `Mozilla/5.0 (compatible; ${BOT}/${VERSION}; +https://pit.world/bot)` + +export type Role = 'site' | 'page' | 'render' | 'publish' + +export type Config = { + redisUrl: string + /** Pages read per site per crawl cycle. */ + maxPages: number + /** Link hops from the home page or a sitemap entry. */ + maxDepth: number + /** Pages fetched at once by one process (each from a different site, by the host lease). */ + concurrency: number + /** Gap between two requests to one site, unless robots.txt asks for more (capped at maxDelayMs). */ + hostDelayMs: number + maxDelayMs: number + /** A site is read again this often; its seen-set and page count expire with it. */ + recrawlDays: number + /** Obscura MCP endpoints, one render at a time each. Empty: no rendering, static HTML only. */ + obscuraUrls: string[] + obscuraToken: string + /** Where records go. Without a key they wait in the Redis outbox (bounded). */ + nichedbUrl: string + nichedbKey: string + /** Follow links to other sites and crawl those too, up to maxSites. */ + discover: boolean + maxSites: number + /** Records waiting for nichedb beyond this are dropped oldest first. */ + outboxMax: number + roles: Role[] + healthPort: number + healthHost: string + userAgent: string +} + +const num = (value: string | undefined, fallback: number, min = 0): number => { + const n = Number(value) + return Number.isFinite(n) && n >= min ? n : fallback +} + +export function configFromEnv(env: Record = process.env): Config { + const roles = (env.CRAWL_ROLES ?? 'site,page,render,publish') + .split(',') + .map((r) => r.trim()) + .filter((r): r is Role => ['site', 'page', 'render', 'publish'].includes(r)) + return { + redisUrl: env.CRAWL_REDIS_URL ?? env.REDIS_URL ?? 'redis://127.0.0.1:6396', + maxPages: num(env.CRAWL_MAX_PAGES, 1000, 1), + maxDepth: num(env.CRAWL_MAX_DEPTH, 6, 0), + concurrency: num(env.CRAWL_CONCURRENCY, 16, 1), + hostDelayMs: num(env.CRAWL_HOST_DELAY_MS, 1500, 0), + maxDelayMs: num(env.CRAWL_MAX_DELAY_MS, 30_000, 0), + recrawlDays: num(env.CRAWL_RECRAWL_DAYS, 30, 1), + obscuraUrls: (env.OBSCURA_MCP_URLS ?? env.OBSCURA_MCP_URL ?? '') + .split(',') + .map((u) => u.trim()) + .filter(Boolean), + obscuraToken: env.OBSCURA_MCP_TOKEN ?? '', + nichedbUrl: (env.NICHEDB_URL ?? 'https://nichedb.dev').replace(/\/+$/, ''), + nichedbKey: env.NICHEDB_API_KEY ?? '', + discover: env.CRAWL_DISCOVER === '1', + maxSites: num(env.CRAWL_MAX_SITES, 100_000, 1), + outboxMax: num(env.CRAWL_OUTBOX_MAX, 200_000, 1), + roles: roles.length ? roles : ['site', 'page', 'render', 'publish'], + healthPort: num(env.CRAWL_HEALTH_PORT, 8797, 0), + // Loopback on a host; 0.0.0.0 inside a container, whose port is published on the host's loopback. + healthHost: env.CRAWL_HEALTH_HOST ?? '127.0.0.1', + userAgent: env.CRAWL_USER_AGENT ?? USER_AGENT, + } +} diff --git a/services/crawler/src/crawler.ts b/services/crawler/src/crawler.ts new file mode 100644 index 0000000..c0328c1 --- /dev/null +++ b/services/crawler/src/crawler.ts @@ -0,0 +1,260 @@ +/** + * What the crawler does, with no queue in it: schedule a site, start a site + * (robots.txt, sitemaps, home page), read a page, render a page, publish. + * The BullMQ wiring (queues.ts) calls these; the tests call them directly + * with an in-memory frontier and a fake web. + */ +import type { Config } from './config.ts' +import { extract, needsRender, type Extracted } from './extract.ts' +import { politeGet, readSitemapXml } from './fetcher.ts' +import type { Frontier } from './frontier.ts' +import type { Rendered } from './obscura.ts' +import { allows, EMPTY, parseRobots, pathOf, type Robots } from './robots.ts' +import { crawlable, normalizeUrl, resolvesPublic, sameSite, siteFromInput, siteOf, type Resolver } from './url.ts' + +export type SiteJob = { site: string; source: string } +export type PageJob = { url: string; site: string; depth: number; rendered?: boolean } + +export type Enqueue = { + sites(jobs: SiteJob[]): Promise + pages(jobs: PageJob[]): Promise + render(job: PageJob): Promise +} + +export type Deps = { + config: Config + frontier: Frontier + enqueue: Enqueue + fetch?: typeof fetch + resolver?: Resolver + renderer?: { size: number; render(url: string): Promise } | null + log?: (line: string) => void +} + +export type PageOutcome = { delayMs?: number; outcome: string } + +/** Links followed from one page; the rest of a link farm waits for another page to mention them. */ +const LINKS_PER_PAGE = 300 +const SITES_PER_PAGE = 25 +const SITEMAP_DOCS = 6 +const BATCH = 200 + +export function createCrawler(deps: Deps) { + const { config, frontier, enqueue } = deps + const resolver = deps.resolver ?? resolvesPublic + const log = deps.log ?? (() => {}) + const get = (url: string, site: string, accept?: string) => + politeGet(url, { userAgent: config.userAgent, resolver, site, ...(deps.fetch ? { fetch: deps.fetch } : {}), ...(accept ? { accept } : {}) }) + + /** Sites pasted, seeded or discovered: checked, counted, queued once per cycle. */ + async function scheduleSites(inputs: string[], source: string): Promise<{ queued: number; refused: number }> { + const jobs: SiteJob[] = [] + let refused = 0 + for (const input of inputs) { + const site = siteFromInput(input) + if (!site || !(await frontier.noteSite(site, config.maxSites, config.recrawlDays * 86_400_000))) { + refused++ + continue + } + jobs.push({ site, source }) + } + if (jobs.length) await enqueue.sites(jobs) + return { queued: jobs.length, refused } + } + + async function robotsFor(site: string): Promise { + const kept = await frontier.getRobots(site) + if (kept) return kept + let robots: Robots = EMPTY + try { + const r = await get(`https://${site}/robots.txt`, site, 'text/plain,*/*;q=0.5') + // 5xx: the site cannot say, so it is read another day (Google's reading). 4xx: no rules. + if (r.status >= 500) return 'unavailable' + if (r.status < 400 && !/html/.test(r.contentType)) robots = parseRobots(r.body) + } catch { + robots = EMPTY + } + await frontier.setRobots(site, robots) + return robots + } + + const gapFor = (robots: Robots) => Math.max(config.hostDelayMs, Math.min(robots.crawlDelayMs ?? 0, config.maxDelayMs)) + + async function admitAll(site: string, urls: string[], depth: number, robots: Robots): Promise { + const jobs: PageJob[] = [] + for (const url of urls) { + if (!crawlable(url) || !allows(robots, pathOf(url))) continue + const verdict = await frontier.admit(site, url) + if (verdict === 'full') break + if (verdict === 'new') jobs.push({ url, site, depth }) + } + return jobs + } + + /** A site's start: robots.txt, then the home page and whatever its sitemaps list, up to maxPages. */ + async function crawlSite(job: SiteJob): Promise<{ admitted: number; outcome: string }> { + const { site } = job + if (!(await resolver(site))) { + await frontier.bump('sites_refused') + await frontier.log({ kind: 'skip', site, note: 'not on the public internet' }) + return { admitted: 0, outcome: 'not public' } + } + const robots = await robotsFor(site) + if (robots === 'unavailable') { + await frontier.bump('sites_unavailable') + await frontier.log({ kind: 'skip', site, note: 'robots.txt answered 5xx; trying another day' }) + return { admitted: 0, outcome: 'robots.txt unavailable' } + } + if (!allows(robots, '/')) { + await frontier.bump('sites_disallowed') + await frontier.log({ kind: 'skip', site, note: 'robots.txt disallows pitbot' }) + return { admitted: 0, outcome: 'robots.txt disallows /' } + } + const home = `https://${site}/` + const listed: string[] = [] + const queue = robots.sitemaps.length ? [...robots.sitemaps] : [`https://${site}/sitemap.xml`] + for (let i = 0; i < queue.length && i < SITEMAP_DOCS && listed.length < config.maxPages; i++) { + const sitemap = queue[i]! + if (!sameSite(sitemap, site)) continue + try { + const r = await get(sitemap, site, 'application/xml,text/xml;q=0.9,*/*;q=0.1') + if (r.status >= 400) continue + const { urls, sitemaps } = readSitemapXml(r.body) + queue.push(...sitemaps.slice(0, SITEMAP_DOCS)) + for (const u of urls) { + const url = normalizeUrl(u) + if (url && sameSite(url, site)) listed.push(url) + } + } catch (error) { + log(`sitemap ${site}: ${(error as Error).message}`) + } + } + const jobs = [...(await admitAll(site, [home], 0, robots)), ...(await admitAll(site, listed.slice(0, config.maxPages), 1, robots))] + if (jobs.length) await enqueue.pages(jobs) + await frontier.bump('sites_started') + await frontier.bump('pages_queued', jobs.length) + await frontier.log({ kind: 'site', site, note: `started: ${jobs.length} pages queued (${listed.length} in sitemaps)` }) + return { admitted: jobs.length, outcome: 'started' } + } + + /** What a read page becomes: a record waiting for nichedb, and the next pages. */ + async function accept(doc: Extracted, job: PageJob, robots: Robots): Promise { + const rendered = doc.enrichment.rendered + if (!doc.noindex) { + await frontier.push([{ record: doc.record, enrichment: doc.enrichment }], config.outboxMax) + await frontier.bump(rendered ? 'pages_rendered' : 'pages_indexed') + } else await frontier.bump('pages_noindex') + let followed = 0 + if (job.depth < config.maxDepth && doc.internal.length) { + const next = await admitAll(job.site, doc.internal.slice(0, LINKS_PER_PAGE), job.depth + 1, robots) + if (next.length) { + await enqueue.pages(next) + await frontier.bump('pages_queued', next.length) + } + followed = next.length + } + await frontier.log({ + kind: doc.noindex ? 'skip' : rendered ? 'render' : 'page', + site: job.site, + url: doc.record.url, + title: doc.record.title, + note: doc.noindex ? 'noindex' : `${doc.enrichment.words} words${followed ? `, ${followed} new links` : ''}`, + }) + if (config.discover && doc.external.length) await scheduleSites(doc.external.slice(0, SITES_PER_PAGE), 'link') + return doc.noindex ? 'noindex' : 'indexed' + } + + async function crawlPage(job: PageJob): Promise { + const robots = await robotsFor(job.site) + if (robots === 'unavailable') return { outcome: 'robots.txt unavailable' } + const wait = await frontier.lease(job.site, gapFor(robots)) + if (wait > 0) return { delayMs: wait, outcome: 'waiting for the site' } + if (!allows(robots, pathOf(job.url))) { + await frontier.bump('pages_disallowed') + return { outcome: 'disallowed' } + } + const r = await get(job.url, job.site) + if (r.offsite) { + if (config.discover) await scheduleSites([r.offsite], 'redirect') + return { outcome: `redirects to ${r.offsite}` } + } + if (r.status >= 400) { + await frontier.bump('pages_failed') + await frontier.log({ kind: 'fail', site: job.site, url: job.url, note: `HTTP ${r.status}` }) + return { outcome: `HTTP ${r.status}` } + } + if (!/html/.test(r.contentType)) return { outcome: `not a page (${r.contentType || 'no type'})` } + const finalUrl = normalizeUrl(r.finalUrl) ?? job.url + const doc = extract(r.body, finalUrl, { depth: job.depth, crawler: config.userAgent }) + if (!job.rendered && deps.renderer?.size && needsRender(r.body, doc.textLength)) { + await enqueue.render({ ...job, rendered: true }) + await frontier.bump('render_queued') + return { outcome: 'needs a browser' } + } + return { outcome: await accept(doc, job, robots) } + } + + async function renderPage(job: PageJob): Promise { + if (!deps.renderer?.size) return { outcome: 'no renderer' } + const robots = await robotsFor(job.site) + if (robots === 'unavailable') return { outcome: 'robots.txt unavailable' } + const wait = await frontier.lease(job.site, gapFor(robots)) + if (wait > 0) return { delayMs: wait, outcome: 'waiting for the site' } + const rendered = await deps.renderer.render(job.url) + const landed = normalizeUrl(rendered.url) ?? job.url + if (!sameSite(landed, job.site)) return { outcome: `rendered onto ${siteOf(landed)}` } + const doc = extract(rendered.html, landed, { depth: job.depth, rendered: true, crawler: config.userAgent }) + return { outcome: await accept(doc, { ...job, rendered: true }, robots) } + } + + /** Records to nichedb, 200 a call, until the outbox is empty or 20 calls are made. */ + async function publish(): Promise<{ sent: number; added: number; rejected: number; waiting: boolean }> { + if (!config.nichedbKey) return { sent: 0, added: 0, rejected: 0, waiting: true } + const fetchImpl = deps.fetch ?? fetch + let sent = 0 + let added = 0 + let rejected = 0 + for (let round = 0; round < 20; round++) { + const raw = await frontier.take(BATCH) + if (!raw.length) break + let response: Response + try { + response = await fetchImpl(`${config.nichedbUrl}/api/v1/sites/records`, { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${config.nichedbKey}`, 'user-agent': config.userAgent }, + body: `{"records":[${raw.join(',')}]}`, + signal: AbortSignal.timeout(60_000), + }) + } catch (error) { + await frontier.giveBack(raw) + throw error + } + if (!response.ok) { + await frontier.giveBack(raw) + throw new Error(`nichedb answered HTTP ${response.status}: ${(await response.text()).slice(0, 200)}`) + } + const body = (await response.json().catch(() => ({}))) as { added?: number; rejected?: unknown[] } + sent += raw.length + added += body.added ?? 0 + rejected += body.rejected?.length ?? 0 + } + if (sent) { + await frontier.bump('published', sent) + await frontier.bump('published_new', added) + if (rejected) await frontier.bump('published_rejected', rejected) + await frontier.log({ kind: 'publish', note: `${sent} records to nichedb: ${added} new${rejected ? `, ${rejected} refused` : ''}` }) + } + return { sent, added, rejected, waiting: false } + } + + /** Sites whose cycle has run out are started again. */ + async function recrawl(limit = 5_000): Promise { + const due = await frontier.sitesDue(config.recrawlDays * 86_400_000, limit) + if (due.length) await scheduleSites(due, 'recrawl') + return due.length + } + + return { scheduleSites, crawlSite, crawlPage, renderPage, publish, recrawl } +} + +export type Crawler = ReturnType diff --git a/services/crawler/src/extract.ts b/services/crawler/src/extract.ts new file mode 100644 index 0000000..a89b736 --- /dev/null +++ b/services/crawler/src/extract.ts @@ -0,0 +1,341 @@ +/** + * One HTML document read the way an index reads it: the OpenSite record + * (logicsrc.com/opensite: the card a careful reader would draw, og/twitter + * tags verbatim, JSON-LD), what the crawler learns on top of it (language, + * keywords, an excerpt of the body text, word and link counts, a hash of the + * text so an unchanged page is not written again), and the links to follow. + * + * Reading order per the spec: JSON-LD, then og:, then twitter:, then HTML. + */ +import { createHash } from 'node:crypto' +import { type HTMLElement, parse } from 'node-html-parser' +import { normalizeUrl, sameSite, siteOf } from './url.ts' + +export type OpenSiteRecord = { + opensite: '0.1' + url: string + canonical: string + site: { name: string; web: string } + kind: string + title: string + description: string | null + image: { url: string; alt?: string } | null + language: string | null + author: { name: string } | null + tags: string[] + feeds: string[] + published_at: string | null + modified_at: string | null + fetched_at: string + status: 'live' | 'gone' | 'moved' | 'blocked' + source: 'read' + cards: { og: Record; twitter: Record } + jsonld: unknown[] +} + +export type Enrichment = { + language: string | null + keywords: string[] + excerpt: string + words: number + links: number + hash: string + rendered: boolean + depth: number + crawler: string +} + +export type Extracted = { + record: OpenSiteRecord + enrichment: Enrichment + /** Same-site links to follow (already normalized), unless nofollow. */ + internal: string[] + /** Sites this page links to. */ + external: string[] + noindex: boolean + nofollow: boolean + /** Visible text length, which decides whether the page needs a browser. */ + textLength: number +} + +const KIND_BY_TYPE: Record = { + article: 'article', + newsarticle: 'article', + blogposting: 'article', + techarticle: 'article', + report: 'article', + product: 'product', + offer: 'product', + person: 'profile', + profilepage: 'profile', + organization: 'profile', + localbusiness: 'profile', + event: 'event', + videoobject: 'video', + 'video.other': 'video', + 'video.movie': 'video', + 'video.episode': 'video', + musicrecording: 'audio', + 'music.song': 'audio', + podcastseries: 'podcast', + podcastepisode: 'episode', + website: 'site', + profile: 'profile', + book: 'product', +} + +/** JSON-LD types that describe a page's own subject, as opposed to its publisher or its site. */ +const CONTENT_TYPES = new Set([ + 'article', + 'newsarticle', + 'blogposting', + 'techarticle', + 'report', + 'product', + 'event', + 'videoobject', + 'musicrecording', + 'podcastseries', + 'podcastepisode', + 'profilepage', + 'person', + 'book', + 'recipe', + 'course', + 'jobposting', + 'softwareapplication', +]) + +/** Small and deliberate: the words that say nothing about a page, English first since most of the web is. */ +const STOP = new Set( + `a about above after again against all also am an and any are as at be because been before being below between both but by can could did do does doing down during each few for from further had has have having he her here hers herself him himself his how i if in into is it its itself just let me more most my myself no nor not now of off on once only or other our ours ourselves out over own same she should so some such than that the their theirs them themselves then there these they this those through to too under until up very was we were what when where which while who whom why will with would you your yours yourself yourselves + www http https com org net html page pages home site website click here read more menu skip content main search privacy policy terms cookie cookies copyright rights reserved contact login sign log subscribe newsletter share email new get one use using used may like via per see us out all just can will also get got make made way back next previous view top bottom free + de la le les des et en un une du pour que qui dans est sur pas plus par avec ce il au ou son sa ses der die das und ist nicht mit sich auf für von zu ein eine el los las del se por con para una al lo como más pero sus le ya o este sí porque esta entre cuando muy sin sobre también me hasta hay donde quien desde todo nos durante todos uno les ni contra otros ese eso ante ellos e esto mí antes algunos qué unos yo otro otras otra él tanto esa estos mucho quienes nada muchos cual poco ella estar estas algunas algo nosotros`.split( + /\s+/, + ), +) + +const clip = (value: string | null | undefined, max: number): string => { + const text = (value ?? '').replace(/\s+/g, ' ').trim() + return text.length > max ? `${text.slice(0, max - 1)}…` : text +} + +function metaMap(root: HTMLElement): { og: Record; twitter: Record; name: Record } { + const og: Record = {} + const twitter: Record = {} + const name: Record = {} + for (const m of root.querySelectorAll('meta')) { + const key = (m.getAttribute('property') ?? m.getAttribute('name') ?? '').trim().toLowerCase() + const content = m.getAttribute('content') + if (!key || content == null) continue + const value = clip(content, 2000) + if (key.startsWith('og:') || key.startsWith('article:')) og[key] ??= value + else if (key.startsWith('twitter:')) twitter[key] ??= value + else name[key] ??= value + } + return { og, twitter, name } +} + +function jsonLdOf(root: HTMLElement): Array> { + const out: Array> = [] + for (const s of root.querySelectorAll('script[type="application/ld+json"]')) { + try { + const parsed = JSON.parse(s.rawText.trim()) + const list = Array.isArray(parsed) ? parsed : Array.isArray(parsed?.['@graph']) ? parsed['@graph'] : [parsed] + for (const entry of list) if (entry && typeof entry === 'object') out.push(entry) + } catch { + // A broken block is somebody else's bug; the page still has its tags. + } + if (out.length >= 8) break + } + return out +} + +const typesOf = (entry: Record): string[] => + ([] as unknown[]).concat(entry['@type'] ?? []).map((t) => String(t).toLowerCase()) + +const str = (v: unknown): string | null => (typeof v === 'string' && v.trim() ? v.trim() : null) + +function dateOf(value: string | null): string | null { + if (!value) return null + const t = Date.parse(value) + return Number.isNaN(t) ? null : new Date(t).toISOString() +} + +/** The words that best say what a page is about: frequency, title words counted thrice, no stopwords, no numbers. */ +export function keywordsOf(title: string, text: string, max = 12): string[] { + const counts = new Map() + const add = (source: string, weight: number) => { + for (const raw of source.toLowerCase().match(/\p{L}[\p{L}\p{M}'’-]{2,29}/gu) ?? []) { + const word = raw.replace(/['’-]+$/g, '') + if (word.length < 3 || STOP.has(word)) continue + counts.set(word, (counts.get(word) ?? 0) + weight) + } + } + add(title, 3) + add(text.slice(0, 50_000), 1) + return [...counts.entries()] + .filter(([, n]) => n >= 2) + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .slice(0, max) + .map(([w]) => w) +} + +/** The language a page declares, as a short tag (en, pt-br), or null. */ +function languageOf(root: HTMLElement, metas: ReturnType): string | null { + const declared = + root.querySelector('html')?.getAttribute('lang') ?? metas.name['content-language'] ?? metas.og['og:locale'] ?? '' + const tag = declared.trim().toLowerCase().replace('_', '-') + return /^[a-z]{2,3}(-[a-z0-9]{2,8})?$/.test(tag) ? tag : null +} + +function visibleText(root: HTMLElement): { all: string; main: string } { + const body = root.querySelector('body') ?? root + for (const el of body.querySelectorAll('script, style, noscript, template, svg, iframe')) el.remove() + const main = body.querySelector('main, article, [role="main"]') + const chrome = (el: HTMLElement) => { + const copy = parse(el.outerHTML) + for (const c of copy.querySelectorAll('nav, header, footer, aside, form, [role="navigation"], [aria-hidden="true"]')) c.remove() + return copy.structuredText + } + const all = body.structuredText.replace(/\s+/g, ' ').trim() + const focused = (main ? chrome(main) : chrome(body)).replace(/\s+/g, ' ').trim() + return { all, main: focused || all } +} + +export function extract( + html: string, + url: string, + options: { fetchedAt?: Date; rendered?: boolean; depth?: number; crawler?: string } = {}, +): Extracted { + const root = parse(html, { comment: false, blockTextElements: { script: true, style: true, noscript: true, pre: true } }) + const metas = metaMap(root) + const jsonld = jsonLdOf(root) + const site = siteOf(url) + const origin = (() => { + try { + return new URL(url).origin + } catch { + return '' + } + })() + + // Canonical: only one on the same site counts, or a page could claim another site's address. + const canonicalHref = root.querySelector('link[rel~="canonical"]')?.getAttribute('href') + const canonicalUrl = canonicalHref ? normalizeUrl(canonicalHref, url) : null + const canonical = canonicalUrl && sameSite(canonicalUrl, site) ? canonicalUrl : url + + // JSON-LD speaks for the page only when it describes the page's own content. The + // publisher's Organization, the WebSite and the breadcrumbs ride on every page of a + // site; taking their name as the title would title every page after the company. + const ld: Record = jsonld.find((e) => typesOf(e).some((t) => CONTENT_TYPES.has(t))) ?? {} + const ldType = typesOf(ld).find((t) => KIND_BY_TYPE[t]) + const ogType = (metas.og['og:type'] ?? '').toLowerCase() + const root_ = new URL(canonical).pathname === '/' + const declared = (ldType && KIND_BY_TYPE[ldType]) || KIND_BY_TYPE[ogType] || null + // og:type=website rides on every page of most sites: only the home page is the site. + const kind = declared && declared !== 'site' ? declared : root_ ? 'site' : 'page' + + const htmlTitle = root.querySelector('title')?.textContent ?? '' + const h1 = root.querySelector('h1')?.textContent ?? '' + const title = clip(str(ld.headline) ?? str(ld.name) ?? metas.og['og:title'] ?? metas.twitter['twitter:title'] ?? (htmlTitle || h1), 300) + const description = + clip(str(ld.description) ?? metas.og['og:description'] ?? metas.twitter['twitter:description'] ?? metas.name.description ?? '', 1000) || null + + const imageRaw = metas.og['og:image'] ?? metas.og['og:image:url'] ?? metas.twitter['twitter:image'] ?? null + const imageUrl = imageRaw ? normalizeUrl(imageRaw, url) : null + const authorName = (() => { + const a = ld.author as unknown + const first = Array.isArray(a) ? a[0] : a + if (first && typeof first === 'object') return str((first as Record).name) + return str(first) ?? metas.name.author ?? null + })() + + const robotsMeta = `${metas.name.robots ?? ''} ${metas.name[`pitbot`] ?? ''}`.toLowerCase() + const noindex = /\bnoindex\b|\bnone\b/.test(robotsMeta) + const nofollow = /\bnofollow\b|\bnone\b/.test(robotsMeta) + + const feeds = root + .querySelectorAll('link[rel~="alternate"]') + .filter((l) => /rss|atom|feed\+json/i.test(l.getAttribute('type') ?? '')) + .map((l) => normalizeUrl(l.getAttribute('href') ?? '', url)) + .filter((u): u is string => Boolean(u)) + .slice(0, 5) + + const internal = new Set() + const external = new Set() + let linkCount = 0 + for (const a of root.querySelectorAll('a[href]')) { + linkCount++ + if (/\bnofollow\b|\bsponsored\b|\bugc\b/i.test(a.getAttribute('rel') ?? '')) continue + const href = normalizeUrl(a.getAttribute('href') ?? '', url) + if (!href) continue + if (sameSite(href, site)) internal.add(href) + else { + const other = siteOf(href) + if (other) external.add(other) + } + } + + const language = languageOf(root, metas) + const { all, main } = visibleText(root) + const words = main ? main.split(/\s+/).length : 0 + const keywords = keywordsOf(title, main) + const fetchedAt = (options.fetchedAt ?? new Date()).toISOString() + + const record: OpenSiteRecord = { + opensite: '0.1', + url, + canonical, + site: { name: clip(metas.og['og:site_name'] ?? site, 120) || site, web: origin }, + kind, + title: title || site, + description, + image: imageUrl ? { url: imageUrl, ...(metas.og['og:image:alt'] ? { alt: clip(metas.og['og:image:alt'], 300) } : {}) } : null, + language, + author: authorName ? { name: clip(authorName, 120) } : null, + tags: keywords.slice(0, 8), + feeds, + published_at: dateOf(str(ld.datePublished) ?? metas.og['article:published_time'] ?? null), + modified_at: dateOf(str(ld.dateModified) ?? metas.og['article:modified_time'] ?? metas.og['og:updated_time'] ?? null), + fetched_at: fetchedAt, + status: 'live', + source: 'read', + cards: { og: metas.og, twitter: metas.twitter }, + jsonld: jsonld.slice(0, 5), + } + + return { + record, + enrichment: { + language, + keywords, + excerpt: clip(main, 1000), + words, + links: linkCount, + hash: createHash('sha1').update(`${title}\n${description ?? ''}\n${main}`).digest('hex'), + rendered: options.rendered === true, + depth: options.depth ?? 0, + crawler: options.crawler ?? 'pitbot', + }, + internal: nofollow ? [] : [...internal], + external: nofollow ? [] : [...external], + noindex, + nofollow, + textLength: all.length, + } +} + +/** + * Whether the HTML is a shell a browser fills in: next to no text, and the + * tell-tales of a client-rendered app (an empty mount point, a bundle, a + * noscript asking for JavaScript). A server-rendered page with little text + * (a landing page) is left alone. + */ +export function needsRender(html: string, textLength: number): boolean { + if (textLength > 600) return false + const mount = /]+id=["'](?:root|app|__next|__nuxt|svelte|main-app|q-app)["'][^>]*>\s*<\/div>/i.test(html) + const asksForJs = /]*>[^<]*(?:enable|requires?|need)[^<]*javascript/i.test(html) + const bundles = (html.match(/]+src=/gi) ?? []).length + return mount || asksForJs || (textLength < 200 && bundles >= 2) +} diff --git a/services/crawler/src/fetcher.ts b/services/crawler/src/fetcher.ts new file mode 100644 index 0000000..f816b3c --- /dev/null +++ b/services/crawler/src/fetcher.ts @@ -0,0 +1,87 @@ +/** + * One polite GET: a named user agent, a timeout, a size cap, and redirects + * followed by hand so every hop is checked against the private-network guard. + */ +import { normalizeUrl, type Resolver } from './url.ts' + +export type Fetched = { + url: string + /** Where the redirects ended. */ + finalUrl: string + status: number + contentType: string + body: string + /** Redirected to another site: not read, handed back so that site can be crawled instead. */ + offsite?: string +} + +type Fetch = typeof fetch + +export const MAX_BYTES = 2 * 1024 * 1024 +const MAX_REDIRECTS = 5 + +async function readCapped(response: Response, max: number): Promise { + if (!response.body) return '' + const reader = response.body.getReader() + const chunks: Uint8Array[] = [] + let size = 0 + while (true) { + const { done, value } = await reader.read() + if (done) break + size += value.byteLength + if (size > max) { + chunks.push(value.subarray(0, value.byteLength - (size - max))) + await reader.cancel().catch(() => {}) + break + } + chunks.push(value) + } + return new TextDecoder('utf-8', { fatal: false }).decode(Buffer.concat(chunks)) +} + +export async function politeGet( + url: string, + options: { userAgent: string; resolver: Resolver; fetch?: Fetch; timeoutMs?: number; accept?: string; site?: string; maxBytes?: number }, +): Promise { + const fetchImpl = options.fetch ?? fetch + let current = url + for (let hop = 0; hop <= MAX_REDIRECTS; hop++) { + const host = new URL(current).hostname + if (!(await options.resolver(host))) throw new Error(`refused: ${host} is not on the public internet`) + const response = await fetchImpl(current, { + redirect: 'manual', + headers: { + 'user-agent': options.userAgent, + accept: options.accept ?? 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.1', + 'accept-language': 'en;q=1, *;q=0.5', + }, + signal: AbortSignal.timeout(options.timeoutMs ?? 20_000), + }) + if (response.status >= 300 && response.status < 400 && response.headers.get('location')) { + await response.body?.cancel().catch(() => {}) + const next = normalizeUrl(response.headers.get('location')!, current) + if (!next) throw new Error('redirect to an address that is not a web page') + const nextSite = new URL(next).hostname.replace(/^www\./, '') + if (options.site && nextSite !== options.site) { + return { url, finalUrl: next, status: response.status, contentType: '', body: '', offsite: nextSite } + } + current = next + continue + } + const contentType = (response.headers.get('content-type') ?? '').toLowerCase() + const textual = /text\/|html|xml|json/.test(contentType) || contentType === '' + const body = textual ? await readCapped(response, options.maxBytes ?? MAX_BYTES) : '' + if (!textual) await response.body?.cancel().catch(() => {}) + return { url, finalUrl: current, status: response.status, contentType, body } + } + throw new Error('too many redirects') +} + +/** Page addresses a sitemap (or sitemap index) lists: { urls, sitemaps }. Plain reading, gzip not handled. */ +export function readSitemapXml(xml: string): { urls: string[]; sitemaps: string[] } { + const locs = (tag: string) => + [...xml.matchAll(new RegExp(`<${tag}\\b[^>]*>[\\s\\S]*?\\s*(?:)?\\s*`, 'gi'))].map((m) => + (m[1] ?? '').replace(/&/g, '&').trim(), + ) + return { urls: locs('url'), sitemaps: locs('sitemap') } +} diff --git a/services/crawler/src/frontier.ts b/services/crawler/src/frontier.ts new file mode 100644 index 0000000..48ba05a --- /dev/null +++ b/services/crawler/src/frontier.ts @@ -0,0 +1,298 @@ +/** + * The crawl's shared state, in Redis so any number of workers on any number + * of boxes agree on it: + * + * pitcrawl:seen: set of url ids admitted this cycle (expires with the cycle) + * pitcrawl:count: pages admitted this cycle; admission stops at maxPages + * pitcrawl:robots: the parsed robots.txt, for a day + * pitcrawl:lease: held for the gap between two requests to one site + * pitcrawl:sites sorted set: site -> when it was last scheduled + * pitcrawl:outbox records waiting for nichedb, oldest first + * pitcrawl:stats counters + * + * Admission is one Lua call, so two workers finding the same link at once + * still admit it once, and the thousand-page cap is exact. + */ +import type { Redis } from 'ioredis' +import { EMPTY, type Robots } from './robots.ts' +import { urlId } from './url.ts' + +/** One line of the crawl log, newest first on /status. */ +export type CrawlEvent = { + at: string + kind: 'site' | 'page' | 'render' | 'skip' | 'fail' | 'publish' + site?: string + url?: string + title?: string + note?: string +} + +/** Counters over time: the last hour, the last day, and the day hour by hour (oldest first). */ +export type Activity = { + lastHour: Record + lastDay: Record + hours: Array<{ at: string; counts: Record }> +} + +const LOG_MAX = 200 +const minuteOf = (t: number) => Math.floor(t / 60_000) +const hourOf = (t: number) => Math.floor(t / 3_600_000) +const add = (into: Record, from: Record) => { + for (const [k, v] of Object.entries(from)) into[k] = (into[k] ?? 0) + Number(v) + return into +} + +export interface Frontier { + /** Admit a page to this cycle: 'new', 'seen', or 'full' (the site has had its maxPages). */ + admit(site: string, url: string): Promise<'new' | 'seen' | 'full'> + /** Take the site's request slot for gapMs; 0 when taken, else how long until it frees. */ + lease(site: string, gapMs: number): Promise + getRobots(site: string): Promise + setRobots(site: string, robots: Robots): Promise + /** + * Remember a site was scheduled now. False (and nothing changes) when it was + * already scheduled within cycleMs, or when the index holds maxSites others. + */ + noteSite(site: string, maxSites: number, cycleMs: number): Promise + sitesDue(olderThanMs: number, limit: number): Promise + push(entries: unknown[], max: number): Promise + take(count: number): Promise + giveBack(raw: string[]): Promise + /** A counter, kept as a total and in minute and hour buckets for rates. */ + bump(field: string, by?: number): Promise + stats(): Promise> + log(event: Omit): Promise + recent(count: number): Promise + activity(): Promise +} + +const K = { + seen: (s: string) => `pitcrawl:seen:${s}`, + count: (s: string) => `pitcrawl:count:${s}`, + robots: (s: string) => `pitcrawl:robots:${s}`, + lease: (s: string) => `pitcrawl:lease:${s}`, + sites: 'pitcrawl:sites', + outbox: 'pitcrawl:outbox', + stats: 'pitcrawl:stats', + minute: (m: number) => `pitcrawl:m:${m}`, + hour: (h: number) => `pitcrawl:h:${h}`, + log: 'pitcrawl:log', +} + +const ADMIT = ` +if redis.call('SADD', KEYS[1], ARGV[1]) == 0 then return 0 end +local n = redis.call('INCR', KEYS[2]) +if n == 1 then + redis.call('PEXPIRE', KEYS[1], ARGV[3]) + redis.call('PEXPIRE', KEYS[2], ARGV[3]) +end +if n > tonumber(ARGV[2]) then return -1 end +return n +` + +type WithAdmit = Redis & { pitcrawlAdmit(seen: string, count: string, id: string, max: number, cycleMs: number): Promise } + +export class RedisFrontier implements Frontier { + private readonly redis: WithAdmit + constructor( + redis: Redis, + private readonly maxPages: number, + private readonly cycleMs: number, + ) { + // A fixed script registered once (EVALSHA after the first call), never built from input. + if (!('pitcrawlAdmit' in redis)) redis.defineCommand('pitcrawlAdmit', { numberOfKeys: 2, lua: ADMIT }) + this.redis = redis as WithAdmit + } + + async admit(site: string, url: string) { + const n = Number(await this.redis.pitcrawlAdmit(K.seen(site), K.count(site), urlId(url), this.maxPages, this.cycleMs)) + return n === 0 ? 'seen' : n < 0 ? 'full' : 'new' + } + + async lease(site: string, gapMs: number) { + if (gapMs <= 0) return 0 + const ok = await this.redis.set(K.lease(site), '1', 'PX', gapMs, 'NX') + if (ok) return 0 + const left = await this.redis.pttl(K.lease(site)) + return Math.max(50, left) + } + + async getRobots(site: string) { + const raw = await this.redis.get(K.robots(site)) + if (!raw) return null + try { + return JSON.parse(raw) as Robots + } catch { + return EMPTY + } + } + + async setRobots(site: string, robots: Robots) { + await this.redis.set(K.robots(site), JSON.stringify(robots), 'EX', 86_400) + } + + async noteSite(site: string, maxSites: number, cycleMs: number) { + const known = await this.redis.zscore(K.sites, site) + if (known !== null && Date.now() - Number(known) < cycleMs) return false + if (known === null && (await this.redis.zcard(K.sites)) >= maxSites) return false + if (known !== null) { + // Due again. Two workers both starting it is harmless: page admission still dedupes. + await this.redis.zadd(K.sites, 'XX', Date.now(), site) + return true + } + // Two workers discovering one new site at once: only the write that lands first schedules it. + return Number(await this.redis.zadd(K.sites, 'NX', Date.now(), site)) === 1 + } + + async sitesDue(olderThanMs: number, limit: number) { + return this.redis.zrangebyscore(K.sites, 0, Date.now() - olderThanMs, 'LIMIT', 0, limit) + } + + async push(entries: unknown[], max: number) { + if (!entries.length) return + await this.redis + .multi() + .rpush(K.outbox, ...entries.map((e) => JSON.stringify(e))) + .ltrim(K.outbox, -max, -1) + .exec() + } + + async take(count: number) { + return (await this.redis.lpop(K.outbox, count)) ?? [] + } + + async giveBack(raw: string[]) { + if (raw.length) await this.redis.lpush(K.outbox, ...[...raw].reverse()) + } + + async bump(field: string, by = 1) { + const now = Date.now() + await this.redis + .multi() + .hincrby(K.stats, field, by) + .hincrby(K.minute(minuteOf(now)), field, by) + .expire(K.minute(minuteOf(now)), 7_200) + .hincrby(K.hour(hourOf(now)), field, by) + .expire(K.hour(hourOf(now)), 49 * 3_600) + .exec() + } + + async stats() { + const [hash, sites, outbox] = await Promise.all([this.redis.hgetall(K.stats), this.redis.zcard(K.sites), this.redis.llen(K.outbox)]) + const out: Record = { sites, outbox } + for (const [k, v] of Object.entries(hash)) out[k] = Number(v) + return out + } + + async log(event: Omit) { + await this.redis + .multi() + .lpush(K.log, JSON.stringify({ at: new Date().toISOString(), ...event })) + .ltrim(K.log, 0, LOG_MAX - 1) + .exec() + } + + async recent(count: number) { + const raw = await this.redis.lrange(K.log, 0, Math.min(count, LOG_MAX) - 1) + return raw.flatMap((line) => { + try { + return [JSON.parse(line) as CrawlEvent] + } catch { + return [] + } + }) + } + + async activity() { + const now = Date.now() + const m = minuteOf(now) + const h = hourOf(now) + const pipe = this.redis.pipeline() + for (let i = 0; i < 60; i++) pipe.hgetall(K.minute(m - i)) + for (let i = 23; i >= 0; i--) pipe.hgetall(K.hour(h - i)) + const results = ((await pipe.exec()) ?? []).map(([, v]) => (v ?? {}) as Record) + const lastHour: Record = {} + for (const r of results.slice(0, 60)) add(lastHour, r) + const hourly = results.slice(60) + const lastDay: Record = {} + for (const r of hourly) add(lastDay, r) + return { + lastHour, + lastDay, + hours: hourly.map((r, i) => ({ at: new Date((h - 23 + i) * 3_600_000).toISOString(), counts: add({}, r) })), + } + } +} + +/** The same contract in memory, for tests and a one-process dry run. */ +export class MemoryFrontier implements Frontier { + seen = new Map>() + leases = new Map() + robots = new Map() + sites = new Map() + outbox: string[] = [] + counters: Record = {} + constructor(private readonly maxPages: number) {} + async admit(site: string, url: string) { + const set = this.seen.get(site) ?? new Set() + this.seen.set(site, set) + const id = urlId(url) + if (set.has(id)) return 'seen' as const + set.add(id) + return set.size > this.maxPages ? ('full' as const) : ('new' as const) + } + async lease(site: string, gapMs: number) { + const until = this.leases.get(site) ?? 0 + if (Date.now() < until) return until - Date.now() + this.leases.set(site, Date.now() + gapMs) + return 0 + } + async getRobots(site: string) { + return this.robots.get(site) ?? null + } + async setRobots(site: string, robots: Robots) { + this.robots.set(site, robots) + } + async noteSite(site: string, maxSites: number, cycleMs: number) { + const known = this.sites.get(site) + if (known !== undefined && Date.now() - known < cycleMs) return false + if (known === undefined && this.sites.size >= maxSites) return false + this.sites.set(site, Date.now()) + return true + } + async sitesDue(olderThanMs: number, limit: number) { + return [...this.sites.entries()] + .filter(([, t]) => t <= Date.now() - olderThanMs) + .slice(0, limit) + .map(([s]) => s) + } + async push(entries: unknown[], max: number) { + this.outbox.push(...entries.map((e) => JSON.stringify(e))) + if (this.outbox.length > max) this.outbox.splice(0, this.outbox.length - max) + } + async take(count: number) { + return this.outbox.splice(0, count) + } + async giveBack(raw: string[]) { + this.outbox.unshift(...raw) + } + events: CrawlEvent[] = [] + async bump(field: string, by = 1) { + this.counters[field] = (this.counters[field] ?? 0) + by + } + async stats() { + return { ...this.counters, sites: this.sites.size, outbox: this.outbox.length } + } + async log(event: Omit) { + this.events.unshift({ at: new Date().toISOString(), ...event }) + this.events.length = Math.min(this.events.length, LOG_MAX) + } + async recent(count: number) { + return this.events.slice(0, count) + } + async activity() { + // In memory everything counted happened "this hour". + const now = new Date(hourOf(Date.now()) * 3_600_000).toISOString() + return { lastHour: { ...this.counters }, lastDay: { ...this.counters }, hours: [{ at: now, counts: { ...this.counters } }] } + } +} diff --git a/services/crawler/src/obscura.ts b/services/crawler/src/obscura.ts new file mode 100644 index 0000000..11391f5 --- /dev/null +++ b/services/crawler/src/obscura.ts @@ -0,0 +1,144 @@ +/** + * Rendering through Obscura (github.com/h4ckf0r0day/obscura), a headless + * browser served as an MCP server (`obscura mcp --http`, bearer token from + * OBSCURA_MCP_TOKEN). One server holds one page, so a client renders one + * address at a time; more rendering means more Obscura servers. + * + * What comes back is a small JSON object, not the page's whole outerHTML: + * a big evaluate result breaks the MCP reply, so the head (where the tags + * live, capped), the visible text and the links are asked for separately + * and put back together as a document the extractor reads like any other. + */ + +export type Rendered = { url: string; html: string } + +type Fetch = typeof fetch + +const EXPRESSION = `(() => { + const d = document; + const links = Array.from(d.querySelectorAll('a[href]')).slice(0, 1500).map((a) => [a.href, a.rel || '']); + return JSON.stringify({ + url: location.href, + lang: d.documentElement.lang || '', + head: d.head ? d.head.outerHTML.slice(0, 120000) : '', + text: d.body ? (d.body.innerText || '').slice(0, 80000) : '', + main: (() => { const m = d.querySelector('main, article, [role="main"]'); return m ? (m.innerText || '').slice(0, 80000) : ''; })(), + links, + }); +})()` + +const esc = (s: string) => s.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"') + +/** The evaluate result as HTML: head verbatim, text as paragraphs, links as anchors. */ +export function documentFrom(parts: { lang?: string; head?: string; text?: string; main?: string; links?: Array<[string, string]> }): string { + const paragraphs = (text: string) => + text + .split(/\n+/) + .map((p) => p.trim()) + .filter(Boolean) + .map((p) => `

${esc(p)}

`) + .join('') + const main = parts.main ? `
${paragraphs(parts.main)}
` : '' + const rest = parts.main ? '' : `
${paragraphs(parts.text ?? '')}
` + const anchors = (parts.links ?? []).map(([href, rel]) => ``).join('') + // Head tags only: a stray
', + }, +} + +const fakeFetch = (async (input: string | URL | Request) => { + const url = String(input instanceof Request ? input.url : input) + const page = WEB[url] + if (!page) return new Response('not found', { status: 404, headers: { 'content-type': 'text/html' } }) + return new Response(page.body, { + status: page.status ?? 200, + headers: { 'content-type': page.type ?? 'text/html', ...(page.location ? { location: page.location } : {}) }, + }) +}) as typeof fetch + +function setup(env: Record = {}) { + const config = configFromEnv({ CRAWL_HOST_DELAY_MS: '0', NICHEDB_URL: 'https://nichedb.test', ...env }) + const frontier = new MemoryFrontier(config.maxPages) + const queued = { sites: [] as SiteJob[], pages: [] as PageJob[], render: [] as PageJob[] } + const crawler = createCrawler({ + config, + frontier, + fetch: fakeFetch, + resolver: async () => true, + renderer: { + size: 1, + async render(url) { + return { url, html: html('Rendered app', 'Client side ovens rendered by a browser, ovens again. A') } + }, + }, + enqueue: { + async sites(jobs) { + queued.sites.push(...jobs) + }, + async pages(jobs) { + queued.pages.push(...jobs) + }, + async render(job) { + queued.render.push(job) + }, + }, + }) + return { config, frontier, queued, crawler } +} + +describe('a site, start to finish', () => { + test('scheduling: once per cycle, never a private name', async () => { + const { crawler, queued } = setup() + expect(await crawler.scheduleSites(['Shop.example', 'shop.example/', 'localhost'], 'cli')).toEqual({ queued: 1, refused: 2 }) + expect(queued.sites).toEqual([{ site: 'shop.example', source: 'cli' }]) + }) + + test('the start reads robots and the sitemap: home and listed pages, never a disallowed or foreign one', async () => { + const { crawler, queued } = setup() + const out = await crawler.crawlSite({ site: 'shop.example', source: 'cli' }) + expect(out.outcome).toBe('started') + expect(queued.pages.map((p) => [p.url, p.depth])).toEqual([ + ['https://shop.example/', 0], + ['https://shop.example/a', 1], + ]) + }) + + test('pages become records and the next pages; a shell goes to the browser; a redirect within the site is followed', async () => { + const { crawler, queued, frontier } = setup() + await crawler.crawlSite({ site: 'shop.example', source: 'cli' }) + expect((await crawler.crawlPage({ url: 'https://shop.example/', site: 'shop.example', depth: 0 })).outcome).toBe('indexed') + const next = queued.pages.slice(2).map((p) => p.url) + expect(next).toEqual(['https://shop.example/b', 'https://shop.example/app']) + // /a was already admitted from the sitemap; /private/y is disallowed. + + expect((await crawler.crawlPage({ url: 'https://shop.example/app', site: 'shop.example', depth: 1 })).outcome).toBe('needs a browser') + expect(queued.render).toEqual([{ url: 'https://shop.example/app', site: 'shop.example', depth: 1, rendered: true }]) + expect((await crawler.renderPage(queued.render[0]!)).outcome).toBe('indexed') + + expect((await crawler.crawlPage({ url: 'https://shop.example/b', site: 'shop.example', depth: 1 })).outcome).toBe('indexed') + const records = frontier.outbox.map((r) => JSON.parse(r)) + expect(records.map((r) => r.record.title)).toEqual(['Shop home', 'Rendered app', 'Page B']) + expect(records[1].enrichment.rendered).toBe(true) + expect(records[2].record.url).toBe('https://www.shop.example/b2') + expect(records[0].enrichment.keywords).toContain('ovens') + // Discovery is off by default: friend.example is not scheduled. + expect(queued.sites).toEqual([]) + // The log /status shows, newest first. + const kinds = frontier.events.map((e) => e.kind) + expect(kinds).toEqual(['page', 'render', 'page', 'site']) + expect(frontier.events[0]).toMatchObject({ site: 'shop.example', url: 'https://www.shop.example/b2', title: 'Page B' }) + expect(frontier.events.at(-1)!.note).toBe('started: 2 pages queued (2 in sitemaps)') + }) + + test('discovery on: linked sites are scheduled too', async () => { + const { crawler, queued } = setup({ CRAWL_DISCOVER: '1' }) + await crawler.crawlPage({ url: 'https://shop.example/', site: 'shop.example', depth: 0 }) + expect(queued.sites).toEqual([{ site: 'friend.example', source: 'link' }]) + }) + + test('the page cap is exact', async () => { + const { crawler, queued } = setup({ CRAWL_MAX_PAGES: '2' }) + await crawler.crawlSite({ site: 'shop.example', source: 'cli' }) + await crawler.crawlPage({ url: 'https://shop.example/', site: 'shop.example', depth: 0 }) + expect(queued.pages.map((p) => p.url)).toEqual(['https://shop.example/', 'https://shop.example/a']) + }) + + test('a site inside its request gap asks to wait, it does not fetch', async () => { + const { crawler } = setup({ CRAWL_HOST_DELAY_MS: '5000' }) + expect((await crawler.crawlPage({ url: 'https://shop.example/', site: 'shop.example', depth: 0 })).outcome).toBe('indexed') + const second = await crawler.crawlPage({ url: 'https://shop.example/a', site: 'shop.example', depth: 1 }) + expect(second.outcome).toBe('waiting for the site') + expect(second.delayMs).toBeGreaterThan(4000) + }) +}) + +describe('publishing to nichedb', () => { + test('without a key, records wait in the outbox', async () => { + const { crawler, frontier } = setup() + await crawler.crawlPage({ url: 'https://shop.example/', site: 'shop.example', depth: 0 }) + expect(await crawler.publish()).toEqual({ sent: 0, added: 0, rejected: 0, waiting: true }) + expect(frontier.outbox.length).toBe(1) + }) + + test('with a key: batches with the bearer, and a failed call puts them back in order', async () => { + const calls: Array<{ url: string; auth: string | null; body: any }> = [] + let fail = true + const nichedb = (async (input: string | URL | Request, init?: RequestInit) => { + const url = String(input) + if (!url.startsWith('https://nichedb.test')) return fakeFetch(input, init) + calls.push({ url, auth: new Headers(init?.headers).get('authorization'), body: JSON.parse(String(init?.body)) }) + if (fail) return new Response('busy', { status: 503 }) + return Response.json({ accepted: 2, added: 2, updated: 0, rejected: [] }) + }) as typeof fetch + const config = configFromEnv({ CRAWL_HOST_DELAY_MS: '0', NICHEDB_URL: 'https://nichedb.test', NICHEDB_API_KEY: 'k1' }) + const frontier = new MemoryFrontier(1000) + const noop = async () => {} + const crawler = createCrawler({ config, frontier, fetch: nichedb, resolver: async () => true, enqueue: { sites: noop, pages: noop, render: noop } }) + await crawler.crawlPage({ url: 'https://shop.example/', site: 'shop.example', depth: 0 }) + await crawler.crawlPage({ url: 'https://shop.example/a', site: 'shop.example', depth: 1 }) + await expect(crawler.publish()).rejects.toThrow('HTTP 503') + expect(frontier.outbox.map((r) => JSON.parse(r).record.title)).toEqual(['Shop home', 'Page A']) + fail = false + expect(await crawler.publish()).toEqual({ sent: 2, added: 2, rejected: 0, waiting: false }) + const last = calls.at(-1)! + expect(last.url).toBe('https://nichedb.test/api/v1/sites/records') + expect(last.auth).toBe('Bearer k1') + expect(last.body.records.map((r: any) => r.record.title)).toEqual(['Shop home', 'Page A']) + expect(frontier.outbox).toEqual([]) + expect(frontier.counters.published).toBe(2) + }) +}) diff --git a/services/crawler/test/parts.test.ts b/services/crawler/test/parts.test.ts new file mode 100644 index 0000000..0316dec --- /dev/null +++ b/services/crawler/test/parts.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, test } from 'bun:test' +import { extract, keywordsOf, needsRender } from '../src/extract.ts' +import { readSitemapXml } from '../src/fetcher.ts' +import { documentFrom } from '../src/obscura.ts' +import { allows, parseRobots } from '../src/robots.ts' +import { crawlable, isPublicAddress, isPublicHostname, normalizeUrl, siteFromInput, siteOf, urlId } from '../src/url.ts' + +describe('addresses', () => { + test('one page, however it is linked', () => { + expect(normalizeUrl('/a?utm_source=x&b=1#top', 'https://Example.com:443/x')).toBe('https://example.com/a?b=1') + expect(normalizeUrl('mailto:a@b.c')).toBeNull() + expect(normalizeUrl('javascript:alert(1)', 'https://example.com/')).toBeNull() + expect(normalizeUrl('https://user:pw@example.com/')).toBeNull() + expect(siteOf('https://www.Example.com/x')).toBe('example.com') + expect(siteFromInput('WWW.example.com/path')).toBe('example.com') + expect(urlId('https://example.com/')).not.toContain(':') + }) + + test('never a private network, by name or by number', () => { + for (const h of ['localhost', '127.0.0.1', '10.1.2.3', '192.168.0.1', '172.20.0.1', '169.254.169.254', 'db.internal', '::1', 'fd00::1', 'foo']) + expect(isPublicHostname(h)).toBe(false) + expect(isPublicHostname('pit.world')).toBe(true) + expect(isPublicAddress('::ffff:127.0.0.1')).toBe(false) + expect(isPublicAddress('23.95.228.174')).toBe(true) + expect(siteFromInput('http://localhost:8787/')).toBeNull() + }) + + test('media, documents and traps are not pages', () => { + expect(crawlable('https://e.com/a.pdf')).toBe(false) + expect(crawlable('https://e.com/wp-login.php')).toBe(true) // a page, but… + expect(crawlable('https://e.com/wp-admin/x')).toBe(false) + expect(crawlable('https://e.com/blog?replytocom=5')).toBe(false) + expect(crawlable('https://e.com/blog/post')).toBe(true) + }) +}) + +describe('robots.txt', () => { + const robots = parseRobots(` +User-agent: * +Disallow: /private +Allow: /private/ok +Disallow: /*.json$ +Crawl-delay: 4 + +User-agent: pitbot +User-agent: other +Disallow: /no-pit + +Sitemap: https://e.com/sitemap.xml +`) + test('our own group wins over *, rules match longest first', () => { + expect(robots.sitemaps).toEqual(['https://e.com/sitemap.xml']) + expect(allows(robots, '/no-pit/x')).toBe(false) + // The * group does not apply once a group names us. + expect(allows(robots, '/private')).toBe(true) + }) + test('the * group: longest match, Allow on ties, wildcards and anchors, crawl-delay', () => { + const star = parseRobots('User-agent: *\nDisallow: /private\nAllow: /private/ok\nDisallow: /*.json$\nCrawl-delay: 4\n') + expect(allows(star, '/private/x')).toBe(false) + expect(allows(star, '/private/ok/y')).toBe(true) + expect(allows(star, '/data.json')).toBe(false) + expect(allows(star, '/data.json?x=1')).toBe(true) + expect(star.crawlDelayMs).toBe(4000) + expect(allows(parseRobots('User-agent: *\nDisallow:\n'), '/anything')).toBe(true) + }) +}) + +const PAGE = ` +HTML title + + + + + + + + + + + +

Pizza

Pizza ovens need heat. A pizza oven made of brick holds heat for hours, and pizza dough loves it.

+Dough Friend Ad
+
Copyright
` + +describe('reading a page', () => { + const doc = extract(PAGE, 'https://www.ovens.example/pizza', { depth: 2 }) + test('the record is the card a reader would draw', () => { + const r = doc.record + expect(r.title).toBe('Wood-fired pizza at home') + expect(r.kind).toBe('article') + // A canonical on another site is a claim, not a fact. + expect(r.canonical).toBe('https://www.ovens.example/pizza') + expect(r.description).toBe('A guide to wood-fired pizza ovens.') + expect(r.image?.url).toBe('https://www.ovens.example/img/oven.jpg') + expect(r.site.name).toBe('Oven Notes') + expect(r.author?.name).toBe('Ann Baker') + expect(r.published_at).toBe('2026-10-01T00:00:00.000Z') + expect(r.feeds).toEqual(['https://www.ovens.example/feed.xml']) + expect(r.cards.twitter['twitter:card']).toBe('summary_large_image') + expect(r.language).toBe('en-us') + }) + test('the enrichment: keywords, excerpt from main, a hash, the links', () => { + expect(doc.enrichment.keywords[0]).toBe('pizza') + expect(doc.enrichment.excerpt).toStartWith('Pizza Pizza ovens need heat.') + expect(doc.enrichment.excerpt).not.toContain('Copyright') + expect(doc.enrichment.hash).toMatch(/^[0-9a-f]{40}$/) + expect(doc.enrichment.depth).toBe(2) + expect(doc.internal).toContain('https://www.ovens.example/recipes/dough') + expect(doc.internal).not.toContain('https://www.ovens.example/ad') + expect(doc.external).toEqual(['friend.example']) + expect(doc.noindex).toBe(false) + }) + test('noindex and nofollow are obeyed', () => { + const d = extract('x', 'https://e.com/') + expect(d.noindex).toBe(true) + expect(d.internal).toEqual([]) + }) + test('keywords skip stopwords, need to recur, and title words count thrice', () => { + expect(keywordsOf('The best ovens', 'ovens ovens the and of bread once')).toEqual(['ovens', 'best']) + }) +}) + +describe('which pages need a browser', () => { + test('an empty app shell does; a short server-rendered page does not', () => { + const shell = '
' + expect(needsRender(shell, extract(shell, 'https://e.com/').textLength)).toBe(true) + expect(needsRender(PAGE, extract(PAGE, 'https://e.com/').textLength)).toBe(false) + const noscript = '' + expect(needsRender(noscript, 40)).toBe(true) + }) + test('what Obscura reads back is a document the extractor reads like any other', () => { + const html = documentFrom({ + lang: 'en', + head: 'App', + text: 'ignored when main exists', + main: 'Line one about \nLine two', + links: [ + ['https://e.com/next', ''], + ['https://e.com/paid', 'sponsored'], + ], + }) + const doc = extract(html, 'https://e.com/', { rendered: true }) + expect(doc.record.title).toBe('Rendered app') + expect(doc.enrichment.excerpt).toBe('Line one about Line two') + expect(doc.enrichment.rendered).toBe(true) + expect(doc.internal).toEqual(['https://e.com/next']) + }) +}) + +test('sitemaps and sitemap indexes', () => { + const index = readSitemapXml('https://e.com/s1.xml') + expect(index).toEqual({ urls: [], sitemaps: ['https://e.com/s1.xml'] }) + const set = readSitemapXml(' https://e.com/b x') + expect(set.urls).toEqual(['https://e.com/a?x=1&y=2', 'https://e.com/b']) +}) + +test("a publisher's JSON-LD never titles the page", () => { + const html = `Topic: example · RSS Amplifier + + + +

x

` + const doc = extract(html, 'https://rssamplifier.com/topics/example') + expect(doc.record.title).toBe('Topic: example') + expect(doc.record.kind).toBe('page') + expect(doc.record.jsonld).toHaveLength(2) +}) diff --git a/services/crawler/test/redis.test.ts b/services/crawler/test/redis.test.ts new file mode 100644 index 0000000..47e48ae --- /dev/null +++ b/services/crawler/test/redis.test.ts @@ -0,0 +1,127 @@ +/** + * The Redis frontier and the BullMQ wiring against a real Redis, and the + * Obscura client against a real Obscura. Each part runs only when its + * server is named, so `bun test` on a laptop passes without either: + * + * CRAWL_TEST_REDIS=redis://127.0.0.1:6399/15 OBSCURA_TEST_URL=http://127.0.0.1:3301/mcp OBSCURA_TEST_TOKEN=… bun test + */ +import { afterAll, describe, expect, test } from 'bun:test' +import { configFromEnv } from '../src/config.ts' +import { extract } from '../src/extract.ts' +import { RedisFrontier } from '../src/frontier.ts' +import { ObscuraClient } from '../src/obscura.ts' +import { connect, createRuntime, snapshot } from '../src/queues.ts' + +const REDIS = process.env.CRAWL_TEST_REDIS +const OBSCURA = process.env.OBSCURA_TEST_URL + +describe.skipIf(!REDIS)('on a real Redis', () => { + const redis = connect(REDIS ?? 'redis://127.0.0.1:1') + afterAll(async () => { + await redis.flushdb() + redis.disconnect() + }) + + test('admission is exact under a race, and the cap holds', async () => { + await redis.flushdb() + const frontier = new RedisFrontier(redis, 5, 60_000) + const urls = Array.from({ length: 20 }, (_, i) => `https://race.example/${i % 8}`) + const verdicts = await Promise.all(urls.map((u) => frontier.admit('race.example', u))) + expect(verdicts.filter((v) => v === 'new').length).toBe(5) + expect(verdicts.filter((v) => v === 'full').length).toBe(3) + expect(await redis.pttl('pitcrawl:seen:race.example')).toBeGreaterThan(0) + }) + + test('the site lease spaces requests; sites are scheduled once per cycle', async () => { + const frontier = new RedisFrontier(redis, 5, 60_000) + expect(await frontier.lease('slow.example', 2_000)).toBe(0) + expect(await frontier.lease('slow.example', 2_000)).toBeGreaterThan(1_000) + const twice = await Promise.all([frontier.noteSite('once.example', 10, 60_000), frontier.noteSite('once.example', 10, 60_000)]) + expect(twice.filter(Boolean).length).toBe(1) + expect(await frontier.noteSite('once.example', 10, 0)).toBe(true) + }) + + test('counters land in the hour and the day; the log is newest first and bounded', async () => { + await redis.flushdb() + const frontier = new RedisFrontier(redis, 5, 60_000) + await frontier.bump('pages_indexed', 3) + await frontier.bump('pages_indexed') + const activity = await frontier.activity() + expect(activity.lastHour.pages_indexed).toBe(4) + expect(activity.lastDay.pages_indexed).toBe(4) + expect(activity.hours).toHaveLength(24) + expect(activity.hours.at(-1)!.counts.pages_indexed).toBe(4) + for (let i = 0; i < 205; i++) await frontier.log({ kind: 'page', note: String(i) }) + const recent = await frontier.recent(500) + expect(recent).toHaveLength(200) + expect(recent[0]!.note).toBe('204') + }) + + test('the outbox keeps order through a give-back and is bounded', async () => { + const frontier = new RedisFrontier(redis, 5, 60_000) + await frontier.push([1, 2, 3, 4], 3) + const taken = await frontier.take(2) + expect(taken).toEqual(['2', '3']) + await frontier.giveBack(taken) + expect(await frontier.take(10)).toEqual(['2', '3', '4']) + }) + + test('queues: page jobs dedupe by address and carry depth as priority; stats answer', async () => { + await redis.flushdb() + const runtime = createRuntime(configFromEnv({ CRAWL_REDIS_URL: REDIS }), () => {}) + try { + expect(await runtime.crawler.scheduleSites(['queue.example'], 'test')).toEqual({ queued: 1, refused: 0 }) + const page = { url: 'https://queue.example/', site: 'queue.example', depth: 0 } + const enqueue = runtime.queues.page + await enqueue.addBulk([{ name: 'page', data: page, opts: { jobId: 'p-x', priority: 1 } }]) + await enqueue.addBulk([{ name: 'page', data: page, opts: { jobId: 'p-x', priority: 1 } }]) + const snap = await snapshot(runtime) + expect(snap.queues.site?.waiting).toBe(1) + expect(snap.queues.page?.prioritized).toBe(1) + expect(snap.stats.sites).toBe(1) + } finally { + await runtime.close() + } + }) +}) + +describe('Obscura over MCP', () => { + test('a fake server: session, navigate, evaluate, a document back', async () => { + const seen: string[] = [] + const fake = (async (_input: string | URL | Request, init?: RequestInit) => { + const body = JSON.parse(String(init?.body)) + const headers = new Headers(init?.headers) + seen.push(`${body.method}${body.params?.name ? `:${body.params.name}` : ''}|${headers.get('authorization')}|${headers.get('mcp-session-id') ?? ''}`) + if (body.method === 'initialize') return Response.json({ jsonrpc: '2.0', id: body.id, result: {} }, { headers: { 'mcp-session-id': 's1' } }) + if (!body.id) return new Response(null, { status: 202 }) + const text = + body.params.name === 'browser_evaluate' + ? JSON.stringify({ url: 'https://spa.example/', lang: 'en', head: 'SPA', text: 'Hello from the client', main: '', links: [['https://spa.example/x', '']] }) + : 'Navigated' + return new Response(`event: message\ndata: ${JSON.stringify({ jsonrpc: '2.0', id: body.id, result: { content: [{ type: 'text', text }] } })}\n\n`, { + headers: { 'content-type': 'text/event-stream' }, + }) + }) as typeof fetch + const client = new ObscuraClient('http://obscura.test/mcp', 'tok', fake) + const out = await client.render('https://spa.example/') + expect(out.url).toBe('https://spa.example/') + expect(out.html).toContain('SPA') + expect(out.html).toContain('

Hello from the client

') + expect(out.html).toContain('href="https://spa.example/x"') + expect(seen).toEqual([ + 'initialize|Bearer tok|', + 'notifications/initialized|Bearer tok|s1', + 'tools/call:browser_navigate|Bearer tok|s1', + 'tools/call:browser_evaluate|Bearer tok|s1', + ]) + }) + + test.skipIf(!OBSCURA)('a real Obscura renders a client-side page', async () => { + const client = new ObscuraClient(OBSCURA!, process.env.OBSCURA_TEST_TOKEN ?? '') + const out = await client.render('https://react.dev/') + const doc = extract(out.html, out.url, { rendered: true }) + expect(doc.record.title).toBe('React') + expect(doc.internal.length).toBeGreaterThan(10) + expect(doc.enrichment.words).toBeGreaterThan(200) + }, 60_000) +}) diff --git a/services/crawler/tsconfig.json b/services/crawler/tsconfig.json new file mode 100644 index 0000000..53e82f9 --- /dev/null +++ b/services/crawler/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2023", + "lib": ["ES2023", "DOM"], + "module": "ESNext", + "moduleResolution": "Bundler", + "allowImportingTsExtensions": true, + "noEmit": true, + "strict": true, + "noUncheckedIndexedAccess": true, + "skipLibCheck": true, + "types": ["bun"] + }, + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/src/crawl-status.ts b/src/crawl-status.ts new file mode 100644 index 0000000..711a9c0 --- /dev/null +++ b/src/crawl-status.ts @@ -0,0 +1,172 @@ +/** + * What pitbot (services/crawler) is doing, for /status and /api/status. + * + * The crawler is its own service; it answers /stats on loopback with its queue + * depths, counters, hour and day rates and a log tail. This reads that and + * turns it into the questions a visitor actually has: is it running, how much + * is waiting, how fast is it going, when does the backlog clear. The web + * server can be healthy while the crawler has been wedged for hours, and that + * failure is invisible from outside, so it gets a page (rssamplifier's + * /crawlstats, same idea). + */ + +export type CrawlEvent = { at: string; kind: string; site?: string; url?: string; title?: string; note?: string } + +export type CrawlerReport = { + ok: boolean + version: string + stats: Record + queues: Record> + activity: { lastHour: Record; lastDay: Record; hours: Array<{ at: string; counts: Record }> } + log: CrawlEvent[] + settings: { maxPages: number; maxDepth: number; concurrency: number; hostDelayMs: number; recrawlDays: number; renderers: number; discover: boolean; publishing: boolean } + generatedAt: string +} + +export type Health = 'healthy' | 'idle' | 'degraded' | 'stalled' | 'offline' + +export type Job = { + key: string + label: string + what: string + state: 'working' | 'idle' | 'stalled' | 'off' + backlog: number + ratePerHour: number + errorsLastHour: number + clearsInHours: number | null +} + +export type CrawlStatus = { + health: Health + summary: string + reachable: boolean + totals: { sites: number; pagesIndexed: number; pagesRendered: number; published: number; outbox: number; failed: number } + lastHour: { pages: number; rendered: number; failed: number; published: number; sitesStarted: number } + lastDay: { pages: number; rendered: number; failed: number; published: number; sitesStarted: number } + jobs: Job[] + /** Pages read per hour, oldest first, for the chart. */ + hourly: Array<{ at: string; pages: number; failed: number }> + log: CrawlEvent[] + settings: CrawlerReport['settings'] | null + generatedAt: string +} + +const n = (o: Record | undefined, ...keys: string[]) => keys.reduce((sum, k) => sum + (Number(o?.[k]) || 0), 0) +const depth = (q: Record | undefined) => n(q, 'waiting', 'prioritized', 'delayed', 'active') + +function job(key: string, label: string, what: string, backlog: number, ratePerHour: number, errorsLastHour: number, enabled = true): Job { + const state: Job['state'] = !enabled ? 'off' : ratePerHour > 0 ? 'working' : backlog > 0 ? 'stalled' : 'idle' + return { + key, + label, + what, + state, + backlog, + ratePerHour, + errorsLastHour, + clearsInHours: backlog > 0 && ratePerHour > 0 ? Math.round((backlog / ratePerHour) * 10) / 10 : null, + } +} + +/** The report as the page shows it. A null report means the crawler did not answer. */ +export function describe(report: CrawlerReport | null, now = new Date()): CrawlStatus { + if (!report) { + return { + health: 'offline', + summary: 'The crawler did not answer. Searches still work; the index just is not growing right now.', + reachable: false, + totals: { sites: 0, pagesIndexed: 0, pagesRendered: 0, published: 0, outbox: 0, failed: 0 }, + lastHour: { pages: 0, rendered: 0, failed: 0, published: 0, sitesStarted: 0 }, + lastDay: { pages: 0, rendered: 0, failed: 0, published: 0, sitesStarted: 0 }, + jobs: [], + hourly: [], + log: [], + settings: null, + generatedAt: now.toISOString(), + } + } + const { stats, queues, activity } = report + const window = (o: Record) => ({ + pages: n(o, 'pages_indexed', 'pages_rendered'), + rendered: n(o, 'pages_rendered'), + failed: n(o, 'pages_failed', 'pages_errored', 'renders_errored'), + published: n(o, 'published'), + sitesStarted: n(o, 'sites_started'), + }) + const lastHour = window(activity.lastHour) + const lastDay = window(activity.lastDay) + const h = activity.lastHour + const fetchedLastHour = n(h, 'pages_indexed', 'pages_failed', 'pages_noindex', 'pages_disallowed', 'render_queued', 'pages_errored') + + const jobs: Job[] = [ + job('sites', 'Site starts', 'robots.txt, sitemaps and the home page of each site', depth(queues.site), n(h, 'sites_started', 'sites_refused', 'sites_disallowed', 'sites_unavailable'), 0), + job('pages', 'Page reads', 'fetch, read and follow same-site links, one request per site at a time', depth(queues.page), fetchedLastHour, n(h, 'pages_failed', 'pages_errored')), + job('render', 'Browser renders', 'pages that are an empty shell until JavaScript runs, read again in Obscura', depth(queues.render), n(h, 'pages_rendered'), n(h, 'renders_errored'), report.settings.renderers > 0), + job('publish', 'Publishing', 'records sent to nichedb.dev/c/sites, two hundred a call', n(stats, 'outbox'), n(h, 'published'), n(h, 'publish_errored'), report.settings.publishing), + ] + + const waiting = jobs.reduce((sum, j) => sum + (j.state === 'off' ? 0 : j.backlog), 0) + const stalled = jobs.filter((j) => j.state === 'stalled' && j.key !== 'publish') + let health: Health + let summary: string + if (stalled.length && fetchedLastHour === 0) { + health = 'stalled' + summary = `${fmt(waiting)} jobs are waiting and nothing has been read in the last hour.` + } else if (report.settings.publishing && jobs[3]!.state === 'stalled') { + health = 'degraded' + summary = `${fmt(n(stats, 'outbox'))} records are waiting for nichedb and none were accepted in the last hour.` + } else if (lastHour.failed > Math.max(20, lastHour.pages)) { + health = 'degraded' + summary = `More pages failed (${fmt(lastHour.failed)}) than were read (${fmt(lastHour.pages)}) in the last hour.` + } else if (waiting === 0 && fetchedLastHour === 0) { + health = 'idle' + summary = 'Nothing is waiting: every site is read for this cycle. New sites and the monthly re-read wake it up.' + } else { + health = 'healthy' + summary = `${fmt(lastHour.pages)} pages read in the last hour, ${fmt(waiting)} jobs waiting.` + } + + return { + health, + summary, + reachable: true, + totals: { + sites: n(stats, 'sites'), + pagesIndexed: n(stats, 'pages_indexed', 'pages_rendered'), + pagesRendered: n(stats, 'pages_rendered'), + published: n(stats, 'published'), + outbox: n(stats, 'outbox'), + failed: n(stats, 'pages_failed', 'pages_errored', 'renders_errored'), + }, + lastHour, + lastDay, + jobs, + hourly: activity.hours.map((x) => ({ at: x.at, pages: n(x.counts, 'pages_indexed', 'pages_rendered'), failed: n(x.counts, 'pages_failed', 'pages_errored', 'renders_errored') })), + log: report.log.slice(0, 60), + settings: report.settings, + generatedAt: report.generatedAt, + } +} + +export function fmt(value: number): string { + return Math.round(value).toLocaleString('en-US') +} + +/** Ask the crawler. Loopback, a short leash, and a few seconds of memory so a busy page does not hammer it. */ +export function createCrawlStatus(url: string, fetchImpl: typeof fetch = fetch, ttlMs = 5_000) { + let cached: { at: number; value: CrawlStatus } | null = null + return async function status(): Promise { + if (cached && Date.now() - cached.at < ttlMs) return cached.value + let report: CrawlerReport | null = null + try { + // threatcrush-disable-next-line js-ssrf-outbound-request fixed loopback address from configuration (CRAWLER_STATUS_URL), never from a request + const response = await fetchImpl(`${url.replace(/\/+$/, '')}/stats`, { signal: AbortSignal.timeout(3_000) }) + if (response.ok) report = (await response.json()) as CrawlerReport + } catch { + report = null + } + const value = describe(report) + cached = { at: Date.now(), value } + return value + } +} diff --git a/src/html.ts b/src/html.ts index 418d1bc..4076a7a 100644 --- a/src/html.ts +++ b/src/html.ts @@ -6,6 +6,7 @@ import type { Listing } from './listings.js' import { createHash } from 'node:crypto' import { readFileSync } from 'node:fs' import { BANGS } from './bangs.js' +import type { CrawlStatus } from './crawl-status.js' import { LINKS, MOSHPIT_ROOT_SHA256, type NetworkStats, type Resolution } from './moshpit.js' import type { Promo } from './promoted.js' import { ringFooter, ringFooterCssUrl, ringFooterOptions, setRingFooterOptions } from './ring-footer.js' @@ -392,7 +393,7 @@ export function privacyPage(): string {

What happens to a search

  1. Your browser sends the words to pit.world over HTTPS.
    2. -
  2. pit.world asks several indexes at once (Google through Serper, Bing through DataForSEO, Brave, Tavily, Ceramic, Exa, Linkup, and our own SearXNG, which asks Bing, Brave, Mojeek, Startpage and Yahoo), takes the top results from each and merges them. Perplexity is asked only when all of those fail. The providers receive the words from our server: not your IP, browser, cookies or referrer.
    3. +
  3. pit.world asks several indexes at once (Google through Serper, Bing through DataForSEO, Brave, Tavily, Ceramic, Exa, Linkup, our own SearXNG, which asks Bing, Brave, Mojeek, Startpage and Yahoo, and our own index, kept at nichedb.dev), takes the top results from each and merges them. Perplexity is asked only when all of those fail. The providers receive the words from our server: not your IP, browser, cookies or referrer.
  4. To avoid paying twice for a popular search, identical searches are answered from memory for up to ten minutes. That cache holds the words and the results, nothing about who asked, never touches disk, and empties on restart.
  5. Result links go straight to the site, with no redirect through us and no-referrer, so the site does not learn what you searched for.
@@ -413,6 +414,108 @@ export function privacyPage(): string { }) } +/** pitbot's live status: is the crawler running, what is waiting, how fast it goes, what it just read. */ +export function statusPage(st: CrawlStatus): string { + const label: Record = { healthy: 'Healthy', idle: 'Idle', degraded: 'Degraded', stalled: 'Stalled', offline: 'Offline' } + const stat = (name: string, value: string, note: string) => `
${esc(name)}${esc(value)}${esc(note)}
` + const ago = (iso: string) => { + const s = Math.max(0, Math.round((Date.now() - Date.parse(iso)) / 1000)) + return s < 60 ? `${s}s ago` : s < 3600 ? `${Math.round(s / 60)}m ago` : `${Math.round(s / 3600)}h ago` + } + const eta = (hours: number | null) => (hours === null ? '' : hours < 1 ? `${Math.max(1, Math.round(hours * 60))} min` : hours < 48 ? `${hours} h` : `${Math.round(hours / 24)} days`) + const max = Math.max(1, ...st.hourly.map((x) => x.pages + x.failed)) + const bars = st.hourly + .map((x, i) => { + const ok = Math.round((x.pages / max) * 60) + const bad = Math.round((x.failed / max) * 60) + return `${esc(new Date(x.at).toISOString().slice(11, 16))} UTC: ${fmt(x.pages)} read, ${fmt(x.failed)} failed` + }) + .join('') + const chart = st.hourly.length + ? `
${bars}
Pages read per hour, last 24 hours (UTC). Failures stacked on top.
` + : '' + const jobs = st.jobs.length + ? `${st.jobs + .map( + (j) => + ``, + ) + .join('')}
JobStateWaitingLast hourErrors (1h)Clears in
${esc(j.label)}
${esc(j.what)}		${esc(j.state)}${fmt(j.backlog)}${fmt(j.ratePerHour)}${fmt(j.errorsLastHour)}${esc(eta(j.clearsInHours))}
` + : '' + const log = st.log.length + ? `
    ${st.log + .map((e) => { + const what = e.url + ? `${esc(e.title || e.url)}` + : esc(e.site ?? '') + const record = e.url && (e.kind === 'page' || e.kind === 'render') ? ` record` : '' + return `
  1. ${esc(e.kind)} ${e.url && e.site ? `${esc(e.site)} ` : ''}${what}${record}${e.note ? ` ${esc(e.note)}` : ''}
    2. ` + }) + .join('')}
` + : '

Nothing logged yet.

' + const set = st.settings + return layout({ + title: 'Crawler status: pitbot', + description: "Live status of pitbot, the crawler behind pit.world's own index: pages read in the last hour and day, what is waiting, and what it just read.", + path: '/status', + bodyClass: 'doc', + head: '', + body: `
${LOGO}pit.world${searchForm()}
+
+

Crawler status

+

Live view of pitbot, the crawler that builds pit.world's own index: up to ${fmt(set?.maxPages ?? 1000)} pages per site, published as open records to nichedb.dev/c/sites and searched next to the other engines. This page refreshes every 15 seconds; the same numbers are at /api/status.

+

${label[st.health]} ${esc(st.summary)}

+${ + st.reachable + ? `
+${stat('Sites', fmt(st.totals.sites), 'known to the crawler')} +${stat('Pages read', fmt(st.totals.pagesIndexed), `${fmt(st.totals.pagesRendered)} through a browser`)} +${stat('Read (1h)', fmt(st.lastHour.pages), `${fmt(st.lastHour.rendered)} rendered`)} +${stat('Read (24h)', fmt(st.lastDay.pages), `${fmt(st.lastDay.sitesStarted)} sites started`)} +${stat('Published', fmt(st.totals.published), `${fmt(st.lastHour.published)} in the last hour`)} +${stat('Waiting to publish', fmt(st.totals.outbox), set?.publishing ? 'records for nichedb' : 'publishing is off')} +${stat('Failed (24h)', fmt(st.lastDay.failed), `${fmt(st.totals.failed)} all time`)} +
+${chart} +

Jobs

+${jobs} +

Live log

+${log} +

${set ? `${fmt(set.concurrency)} pages at once per worker, ${fmt(set.hostDelayMs / 1000)} s between requests to one site, ${fmt(set.renderers)} browsers, ${fmt(set.maxDepth)} link hops, every site re-read each ${fmt(set.recrawlDays)} days${set.discover ? ', following links to new sites' : ''}. ` : ''}Generated ${esc(ago(st.generatedAt))}.

` + : '' +} +
`, + }) +} + +/** pitbot: who reads your site for pit.world's own index, and how to say no. */ +export function botPage(): string { + return layout({ + title: 'pitbot: the pit.world crawler', + description: 'pitbot reads up to a thousand pages per site for pit.world\'s own search index. It obeys robots.txt, noindex and nofollow, and waits between requests.', + path: '/bot', + bodyClass: 'doc', + body: `
${LOGO}pit.world${searchForm()}
+
+

pitbot, the pit.world crawler

+

pitbot reads pages for pit.world's own search index. What it reads is published as open OpenSite records at nichedb.dev/c/sites, one per page, and pit.world searches them alongside the other indexes it asks.

+

How it behaves

+
    +
  • It identifies itself: Mozilla/5.0 (compatible; pitbot/0.1; +https://pit.world/bot).
    • +
  • It obeys robots.txt: a group for pitbot if you write one, else *, and Crawl-delay up to 30 seconds.
    • +
  • It obeys <meta name="robots" content="noindex"> and nofollow, and does not follow rel="nofollow", sponsored or ugc links.
    • +
  • It sends one request at a time to a site, with a pause between them, and reads at most a thousand pages per site per month.
    • +
  • Pages that are an empty shell until JavaScript runs are read once more in a headless browser, so the index sees what a visitor sees.
    • +
  • It reads only public pages: no forms, no logins, no carts.
    • +
+

Keep it out

+
User-agent: pitbot
+Disallow: /
+

A page already in the index is dropped on its next read. To ask for anything else, write to riotcoder@profullstack.com.

+
`, + }) +} + export function aboutPage(): string { const bangRows = BANGS.map((b) => `!${esc(b.aliases?.[0])}${(b.aliases ?? []).slice(1).map((a) => ` !${esc(a)}`).join('')}${esc(b.name)}`).join('') return layout({ diff --git a/src/search.ts b/src/search.ts index 0496aba..1b666f4 100644 --- a/src/search.ts +++ b/src/search.ts @@ -18,7 +18,7 @@ export type Hit = { url: string; title: string; description: string; host: string; sources?: Provider[] } -export type Provider = 'serper' | 'bing' | 'brave' | 'tavily' | 'ceramic' | 'exa' | 'linkup' | 'searxng' | 'perplexity' +export type Provider = 'serper' | 'bing' | 'brave' | 'tavily' | 'ceramic' | 'exa' | 'linkup' | 'searxng' | 'pit' | 'perplexity' export type Keys = Partial> @@ -32,7 +32,7 @@ export type SearchResult = { cached: boolean } /** Asked together and merged. Broad indexes first: fusion ties go to the earlier list, and SearXNG already blends five engines. */ -export const PRIMARY: Provider[] = ['serper', 'bing', 'searxng', 'brave', 'tavily', 'exa', 'linkup', 'ceramic'] +export const PRIMARY: Provider[] = ['serper', 'bing', 'searxng', 'brave', 'tavily', 'exa', 'linkup', 'ceramic', 'pit'] /** Tried one by one, only when every primary failed. */ export const FALLBACK: Provider[] = ['perplexity'] export const ORDER: Provider[] = [...PRIMARY, ...FALLBACK] @@ -51,6 +51,7 @@ export const LABELS: Record = { exa: 'Exa', linkup: 'Linkup', searxng: 'SearXNG', + pit: 'pit.world index', perplexity: 'Perplexity', } export const MAX_QUERY = 512 @@ -152,6 +153,16 @@ function stripTags(value: unknown): string { return typeof value === 'string' ? value.replace(/<[^>]*>/g, '') : '' } +/** + * Our own index: pages pit.world's crawler (services/crawler) read and + * published to nichedb.dev/c/sites, searched there. Title, summary (the page's + * description or an excerpt of its text) and keyword tags are what match. + */ +export function parsePitIndex(body: any): Hit[] { + const items: any[] = Array.isArray(body?.items) ? body.items : [] + return cleanHits(items.map((i) => ({ url: i?.url, title: i?.title, description: i?.summary }))) +} + export function parsePerplexity(body: any): Hit[] { const results: any[] = Array.isArray(body?.results) ? body.results : [] return cleanHits(results.map((r) => ({ url: r?.url, title: r?.title, description: r?.snippet }))) @@ -215,6 +226,17 @@ export async function callProvider(provider: Provider, key: string, query: strin case 'searxng': // key is the instance's base URL, e.g. http://127.0.0.1:8888, reached over loopback. return parseSearxng(await getJson(fetchImpl, `${key.replace(/\/+$/, '')}/search?${new URLSearchParams({ q: query, format: 'json' })}`, {})).slice(0, limit) + case 'pit': { + // key is "[ ]": the index is public, a key only lifts the rate limit. + const [base = '', token] = key.trim().split(/\s+/) + return parsePitIndex( + await getJson( + fetchImpl, + `${base.replace(/\/+$/, '')}/api/v1/search?${new URLSearchParams({ q: query, collection: 'sites', limit: String(limit) })}`, + token ? { authorization: `Bearer ${token}` } : {}, + ), + ).slice(0, limit) + } case 'linkup': return parseLinkup( await postJson(fetchImpl, 'https://api.linkup.so/v1/search', { authorization: `Bearer ${key}` }, { q: query, depth: 'standard', outputType: 'searchResults' }), diff --git a/src/server.ts b/src/server.ts index 0bc0811..b29b2ff 100644 --- a/src/server.ts +++ b/src/server.ts @@ -4,10 +4,11 @@ * Nothing here writes a log line about a request. Errors are reported without * the query attached. */ +import { type CrawlStatus, createCrawlStatus } from './crawl-status.js' import { FOOTER_SCRIPT_SRC, ringFooterCss, startRingFooterRefresh } from './ring-footer.js' import { Hono, type Context } from 'hono' import { parseBang, BANGS } from './bangs.js' -import { aboutPage, buildPage, startPage, connectPage, errorPage, homePage, mediaPage, mobileConfig, newsletterPage, offlinePage, subscribersAdminPage, openSearchXml, privacyPage, resultsPage, ringOpml, ringPage } from './html.js' +import { aboutPage, botPage, statusPage, buildPage, startPage, connectPage, errorPage, homePage, mediaPage, mobileConfig, newsletterPage, offlinePage, subscribersAdminPage, openSearchXml, privacyPage, resultsPage, ringOpml, ringPage } from './html.js' import { adminAuthorized, brandFor, Newsletter, normalizeSource, resendMailer } from './newsletter.js' import { createFormGuard } from '@profullstack/form-guard' import { looksLikeSpam, orderByInvoice, OrderError, readOrder, type BuildInvoicing } from './build-invoice.js' @@ -52,6 +53,8 @@ export type Deps = { newsletter?: Newsletter /** Bearer token for /api/admin/*; unset means the admin API is off. */ adminToken?: string + /** pitbot's status for /status; defaults to the crawler's loopback /stats (CRAWLER_STATUS_URL). */ + crawlStatus?: () => Promise /** PairUX + nixamp listings for the Videos, Music and Live tabs. */ promoted?: Promoted /** Claimable business listings (/b); unset means the feature is off. */ @@ -93,6 +96,7 @@ export function createApp(deps: Deps) { const cache = deps.cache ?? new Cache() const limiter = deps.limiter ?? new RateLimiter() const promoted = deps.promoted ?? new Promoted(deps.fetch) + const crawlStatus = deps.crawlStatus ?? createCrawlStatus(process.env.CRAWLER_STATUS_URL ?? 'http://127.0.0.1:8807') const app = new Hono() // Training crawlers buy a day pass before anything else, and ahead of the header @@ -219,6 +223,17 @@ export function createApp(deps: Deps) { }) app.get('/privacy', (c) => c.html(privacyPage())) app.get('/about', (c) => c.html(aboutPage())) + app.get('/bot', (c) => c.html(botPage())) + // pitbot's live status: the crawler is its own service, read over loopback. + app.get('/status', async (c) => { + c.header('cache-control', 'no-store') + return c.html(statusPage(await crawlStatus())) + }) + app.get('/api/status', async (c) => { + c.header('cache-control', 'no-store') + c.header('access-control-allow-origin', '*') + return c.json(await crawlStatus()) + }) app.get('/connect', (c) => c.html(connectPage())) // The house form guard on the invoice form: a signed token minted with the page (a bot that POSTs // without rendering /build has none), a fill-time floor and the honeypot. Rate limiting stays ours. @@ -489,6 +504,8 @@ export function keysFromEnv(env: Record = process.en exa: env.EXA_API_KEY, // Not a key: the base URL of a SearXNG instance on loopback, e.g. http://127.0.0.1:8888. searxng: env.SEARXNG_URL, + // Not a key either: our own index on nichedb, "[ ]". PIT_INDEX_URL=off turns it off. + pit: env.PIT_INDEX_URL === 'off' ? undefined : `${env.PIT_INDEX_URL ?? 'https://nichedb.dev'}${env.PIT_INDEX_KEY ? ` ${env.PIT_INDEX_KEY}` : ''}`, perplexity: env.PERPLEXITY_API_KEY, } } diff --git a/test/index.test.ts b/test/index.test.ts new file mode 100644 index 0000000..ca87b59 --- /dev/null +++ b/test/index.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from 'bun:test' +import { createCrawlStatus, type CrawlerReport, describe as describeCrawl } from '../src/crawl-status.ts' +import { callProvider, LABELS, parsePitIndex, PRIMARY } from '../src/search.ts' +import { createApp, keysFromEnv } from '../src/server.ts' + +describe('our own index as one of the engines', () => { + it('reads nichedb search items: url, title, summary; never a non-web link', () => { + expect( + parsePitIndex({ + items: [ + { url: 'javascript:alert(1)', title: 'bad' }, + { url: 'https://ovens.example/pizza', title: 'Wood-fired pizza', summary: 'How to build one.' }, + ], + }), + ).toEqual([{ url: 'https://ovens.example/pizza', title: 'Wood-fired pizza', description: 'How to build one.', host: 'ovens.example' }]) + }) + + it('asks the sites collection, with a bearer only when one is configured', async () => { + const asked: Array<{ url: string; auth: string | null }> = [] + const fake = (async (input: string | URL | Request, init?: RequestInit) => { + asked.push({ url: String(input), auth: new Headers(init?.headers).get('authorization') }) + return Response.json({ items: [{ url: 'https://a.example/', title: 'A', summary: '' }] }) + }) as typeof fetch + expect(await callProvider('pit', 'https://nichedb.test/', 'pizza ovens', 10, fake)).toHaveLength(1) + await callProvider('pit', 'https://nichedb.test k1', 'x', 5, fake) + expect(asked[0]).toEqual({ url: 'https://nichedb.test/api/v1/search?q=pizza+ovens&collection=sites&limit=10', auth: null }) + expect(asked[1]!.auth).toBe('Bearer k1') + }) + + it('is on by default, last among the primaries, and can be turned off', () => { + expect(PRIMARY.at(-1)).toBe('pit') + expect(LABELS.pit).toBe('pit.world index') + expect(keysFromEnv({}).pit).toBe('https://nichedb.dev') + expect(keysFromEnv({ PIT_INDEX_URL: 'http://127.0.0.1:3010', PIT_INDEX_KEY: 'k' }).pit).toBe('http://127.0.0.1:3010 k') + expect(keysFromEnv({ PIT_INDEX_URL: 'off' }).pit).toBeUndefined() + }) + + it('/bot says who pitbot is and how to keep it out', async () => { + const app = createApp({ keys: {} }) + const page = await (await app.request('/bot')).text() + expect(page).toContain('User-agent: pitbot') + expect(page).toContain('nichedb.dev/c/sites') + expect(await (await app.request('/privacy')).text()).toContain('href="/bot"') + }) +}) + +const report = (over: Partial = {}): CrawlerReport => ({ + ok: true, + version: '0.1.0', + stats: { sites: 3, outbox: 12, pages_indexed: 40, pages_rendered: 2, published: 30 }, + queues: { site: { waiting: 0 }, page: { prioritized: 500, delayed: 20 }, render: { waiting: 1 }, publish: { delayed: 2 } }, + activity: { + lastHour: { pages_indexed: 30, pages_rendered: 2, published: 30, pages_failed: 1 }, + lastDay: { pages_indexed: 40, pages_rendered: 2, published: 30, sites_started: 3, pages_failed: 1 }, + hours: [{ at: '2026-11-02T04:00:00.000Z', counts: { pages_indexed: 30, pages_failed: 1 } }], + }, + log: [{ at: '2026-11-02T04:35:00.000Z', kind: 'page', site: 'shop.example', url: 'https://shop.example/a', title: 'A page', note: '300 words' }], + settings: { maxPages: 1000, maxDepth: 6, concurrency: 16, hostDelayMs: 1500, recrawlDays: 30, renderers: 2, discover: false, publishing: true }, + generatedAt: new Date().toISOString(), + ...over, +}) + +describe('crawler status', () => { + it('healthy, stalled, idle, offline: the verdict a visitor needs', () => { + const ok = describeCrawl(report()) + expect(ok.health).toBe('healthy') + expect(ok.jobs.find((j) => j.key === 'pages')).toMatchObject({ state: 'working', backlog: 520, clearsInHours: 16.8 }) + const stuck = describeCrawl(report({ activity: { lastHour: {}, lastDay: {}, hours: [] } })) + expect(stuck.health).toBe('stalled') + const idle = describeCrawl(report({ queues: {}, stats: { sites: 3 }, activity: { lastHour: {}, lastDay: {}, hours: [] } })) + expect(idle.health).toBe('idle') + expect(describeCrawl(null).health).toBe('offline') + }) + + it('/status draws it, escaped, refreshing; /api/status is the same as JSON', async () => { + const app = createApp({ keys: {}, crawlStatus: async () => describeCrawl(report()) }) + const page = await (await app.request('/status')).text() + expect(page).toContain('Crawler status') + expect(page).toContain('') + expect(page).toContain('A <b>page</b>') + expect(page).toContain('https://nichedb.dev/c/sites/shop.example/a') + expect(page).toContain('crawl-health-healthy') + const json = await (await app.request('/api/status')).json() + expect(json.totals.pagesIndexed).toBe(42) + expect(json.health).toBe('healthy') + }) + + it('a crawler that does not answer is shown as offline, not as an error page', async () => { + const down = createCrawlStatus('http://127.0.0.1:9', (async () => { + throw new Error('refused') + }) as unknown as typeof fetch) + const app = createApp({ keys: {}, crawlStatus: down }) + const r = await app.request('/status') + expect(r.status).toBe(200) + expect(await r.text()).toContain('crawl-health-offline') + }) +})