Skip to content

Commit 824263f

Browse files
ralyodioclaude
andauthored
nichedb-nightly: a Traffic section, so the report is not all zeros (#23)
Anthony (bot@ queue, 2026-10-16): "can we track anything here? its all zeros?". The money lines are zero because nothing has been bought, but the API rate limiter's hourly counter (api_usage) already records every /api request per caller: ~270K requests a day from ~190K addresses. The report now shows that (internal proxy addresses counted apart), plus accounts, sign-in links sent vs used, API keys and feed alerts delivered, and the closing note says the traffic is real and unpaid. The ssh read timeout goes to 10 min: the whole reading took 4m47s in the daytime against a 5 min limit. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 493caa8 commit 824263f

1 file changed

Lines changed: 37 additions & 2 deletions

File tree

‎bin/nichedb-nightly‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,33 @@ r.index = await one(sql${'`'}
129129
(select count(*) from items where first_seen_at > now() - interval '24 hours')::bigint as items_24h,
130130
(select count(*) from collections)::int as collections${'`'});
131131
132+
// Demand that exists before anyone pays. api_usage is the API rate limiter's
133+
// hourly counter (bucket ip:<addr> or key:<id>, kept 7 days), so it is every
134+
// /api request, crawlers included. Private addresses are our own proxies and
135+
// services, counted apart so they do not pass for visitors.
136+
r.traffic = await one(sql${'`'}
137+
with u as (
138+
select bucket, hour, count,
139+
bucket like 'key:%' as keyed,
140+
bucket ~ '^ip:(10\.|127\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.)' as internal
141+
from api_usage where hour >= now() - interval '7 days')
142+
select coalesce(sum(count) filter (where hour >= now() - interval '24 hours' and not internal),0)::bigint as reqs_24h,
143+
count(distinct bucket) filter (where hour >= now() - interval '24 hours' and not internal and not keyed)::int as ips_24h,
144+
count(distinct bucket) filter (where hour >= now() - interval '24 hours' and keyed)::int as keys_24h,
145+
coalesce(sum(count) filter (where hour >= now() - interval '24 hours' and keyed),0)::bigint as keyed_reqs_24h,
146+
coalesce(sum(count) filter (where hour >= now() - interval '24 hours' and internal),0)::bigint as internal_24h,
147+
coalesce(sum(count) filter (where not internal),0)::bigint as reqs_7d,
148+
greatest(1, ceil(extract(epoch from now() - min(hour)) / 86400))::int as days
149+
from u${'`'});
150+
r.people = await one(sql${'`'}
151+
select (select count(*) from users)::int as users,
152+
(select count(*) from users where created_at > now() - interval '24 hours')::int as new_24h,
153+
(select count(*) from users where last_seen_at > now() - interval '24 hours')::int as seen_24h,
154+
(select count(*) from login_tokens where created_at > now() - interval '24 hours')::int as logins_24h,
155+
(select count(*) from login_tokens where created_at > now() - interval '24 hours' and consumed_at is not null)::int as logins_used_24h,
156+
(select count(*) from api_keys)::int as api_keys,
157+
(select count(*) from deliveries where sent_at > now() - interval '24 hours')::int as alerts_24h${'`'});
158+
132159
// bigint columns arrive as BigInt or strings; every one of them fits a double.
133160
const plain = (v) => typeof v === 'bigint' ? Number(v)
134161
: Array.isArray(v) ? v.map(plain)
@@ -144,7 +171,7 @@ function read() {
144171
let out;
145172
try {
146173
out = execFileSync('ssh', ['-o', 'BatchMode=yes', '-o', 'ConnectTimeout=30', '-i', SSH_KEY, APP_HOST, remote],
147-
{ input: STATS, encoding: 'utf8', timeout: 300_000, maxBuffer: 16 << 20, stdio: ['pipe', 'pipe', 'pipe'] });
174+
{ input: STATS, encoding: 'utf8', timeout: 600_000, maxBuffer: 16 << 20, stdio: ['pipe', 'pipe', 'pipe'] });
148175
} catch (e) {
149176
throw new Error(`the stats script failed in ${APP_CONTAINER} on ${APP_HOST}: ${String(e.stderr || e.message).trim().slice(-400)}`);
150177
}
@@ -215,7 +242,8 @@ function todoList(r) {
215242
/** The honest closing line. A page of zeros should not read as a fault. */
216243
function honestNote(r) {
217244
if (r.sales.n === 0 && r.rev.events === 0)
218-
return 'No crawl pass has ever been bought and the ledger is empty, so every money line is zero by circumstance rather than by fault. The paywall is live and answering 402.';
245+
return 'No crawl pass has ever been bought and the ledger is empty, so every money line is zero by circumstance rather than by fault. The paywall is live and answering 402.' +
246+
(r.traffic?.reqs_24h ? ` The traffic is real: ${num(r.traffic.reqs_24h)} API requests from ${num(r.traffic.ips_24h)} addresses in 24h, mostly crawlers, and none of it paid.` : '');
219247
if (r.alloc.owed === 0 && r.members.active === 0)
220248
return "Money is arriving but nobody operates a niche, so all of it is the platform's. A claimed niche is what starts splitting it.";
221249
return null;
@@ -225,6 +253,13 @@ function honestNote(r) {
225253
function sections(r) {
226254
const q = r.questions;
227255
return [
256+
...(r.traffic ? [{ title: 'Traffic', sub: 'demand before money: API requests and accounts', rows: [
257+
['API, 24h', `${num(r.traffic.reqs_24h)} requests from ${plural(r.traffic.ips_24h, 'address', 'addresses')}; ${plural(r.traffic.keys_24h, 'API key')} used (${num(r.traffic.keyed_reqs_24h)} requests)`],
258+
['API, 7d', `${num(r.traffic.reqs_7d)} requests, ${num(Math.round(r.traffic.reqs_7d / Math.max(1, Math.min(7, r.traffic.days))))} a day on average`],
259+
['Internal', `${num(r.traffic.internal_24h)} requests from our own proxies and services (not counted above)`],
260+
['Accounts', `${num(r.people.users)} total, ${num(r.people.new_24h)} new and ${num(r.people.seen_24h)} seen in 24h; ${plural(r.people.logins_24h, 'sign-in link')} sent, ${num(r.people.logins_used_24h)} used; ${plural(r.people.api_keys, 'API key')} issued`],
261+
['Alerts', `${plural(r.people.alerts_24h, 'feed alert')} delivered in 24h`],
262+
] }] : []),
228263
{ title: 'Niches', rows: [
229264
['Total', `${num(r.niches.total)}: ${num(r.niches.operated)} operated, ${num(r.niches.open)} open for a claim`],
230265
['Members', `${plural(r.members.active, 'active membership')} across ${plural(r.members.people, 'person', 'people')}`],

0 commit comments

Comments
 (0)