diff --git a/apps/cli/README.md b/apps/cli/README.md index 2cf229dc..ae7d3d32 100644 --- a/apps/cli/README.md +++ b/apps/cli/README.md @@ -165,6 +165,15 @@ threatcrush init --offline # Same, without signing in (scripts, CI, con Config lives at `/etc/threatcrush/threatcrushd.conf` with module configs in `/etc/threatcrush/threatcrushd.conf.d/`. +### Usage counts + +Each `threatcrush ai-scan` / `ai-scan-run` sends one anonymous count to the API +(`THREATCRUSH_API_URL`, default threatcrush.com): command, CLI version, OS, Node +version, CI or not, a random install ID (`~/.threatcrush/usage-ping.json`), and +probe/finding counts and run time. Never the target, prompts, responses or +finding text. Turn it off with `THREATCRUSH_TELEMETRY=0` or `DO_NOT_TRACK=1`. +Requests to our API also carry the user agent `threatcrush/`. + ## Cloud dashboard The daemon reports to your ThreatCrush dashboard once the machine is logged in and linked to a server there: diff --git a/apps/cli/src/commands/ai-scan.ts b/apps/cli/src/commands/ai-scan.ts index f445fa41..830c8e05 100644 --- a/apps/cli/src/commands/ai-scan.ts +++ b/apps/cli/src/commands/ai-scan.ts @@ -6,6 +6,7 @@ import type { RunResult } from '../core/run-result.js'; import { runAiScan, httpSender } from '../aiscan/runner.js'; import { selectProbes, type ProbeCategory } from '../aiscan/probes.js'; import { runMosh } from '../aiscan/mosh.js'; +import { sendUsagePing } from '../core/usage-ping.js'; export interface AiScanCliOptions { categories?: string; // comma list @@ -46,6 +47,7 @@ export async function aiScanCommand(url: string, opts: AiScanCliOptions): Promis const probes = selectProbes(categories); logger.info(`Probing ${chalk.white(url)} with ${probes.length} check(s)...\n`); const spinner = ora({ text: 'Running probes...', color: 'green' }).start(); + const t0 = Date.now(); const result = await runAiScan(url, send, { categories, @@ -55,6 +57,10 @@ export async function aiScanCommand(url: string, opts: AiScanCliOptions): Promis }, }); spinner.stop(); + await sendUsagePing({ + event: 'ai-scan', probes: probes.length, findings: result.findings.length, + severity: result.severity_summary, duration_ms: Date.now() - t0, failed: Boolean(result.error), + }); if (opts.json) { console.log(JSON.stringify(result, null, 2)); @@ -93,7 +99,12 @@ export async function aiScanMoshCommand(file: string, args: string[]): Promise console.log(' ' + s) }); + await sendUsagePing({ + event: 'ai-scan-run', scans: res.scans.length, findings: res.scans.reduce((n, s) => n + s.findings.length, 0), + duration_ms: Date.now() - t0, failed: Boolean(res.failed), + }); if (res.failed) { console.log(); console.log(` ${chalk.red('✗ script failed:')} ${res.failed}`); diff --git a/apps/cli/src/core/__tests__/usage-ping.test.ts b/apps/cli/src/core/__tests__/usage-ping.test.ts new file mode 100644 index 00000000..f4be7a85 --- /dev/null +++ b/apps/cli/src/core/__tests__/usage-ping.test.ts @@ -0,0 +1,68 @@ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { installId, sendUsagePing, usagePingDisabled } from "../usage-ping.js"; + +const dirs: string[] = []; +const tmp = () => { const d = mkdtempSync(join(tmpdir(), "tc-ping-")); dirs.push(d); return d; }; +afterEach(() => { for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }); }); + +function capture() { + const calls: { url: string; body: Record }[] = []; + const fetchImpl = (async (url: string, init?: RequestInit) => { + calls.push({ url, body: JSON.parse(String(init?.body)) }); + return new Response("{}", { status: 202 }); + }) as unknown as typeof fetch; + return { calls, fetchImpl }; +} + +describe("usage ping", () => { + it("is off with THREATCRUSH_TELEMETRY=0/off/false/no or DO_NOT_TRACK=1", () => { + for (const v of ["0", "off", "false", "no", "OFF"]) expect(usagePingDisabled({ THREATCRUSH_TELEMETRY: v })).toBe(true); + expect(usagePingDisabled({ DO_NOT_TRACK: "1" })).toBe(true); + expect(usagePingDisabled({ DO_NOT_TRACK: "0" })).toBe(false); + expect(usagePingDisabled({})).toBe(false); + }); + + it("sends nothing when disabled", async () => { + const { calls, fetchImpl } = capture(); + const ok = await sendUsagePing({ event: "ai-scan", findings: 1, duration_ms: 5 }, { fetchImpl, dir: tmp(), env: { DO_NOT_TRACK: "1" } }); + expect(ok).toBe(false); + expect(calls).toHaveLength(0); + }); + + it("keeps one install id and prints the notice only the first time", async () => { + const dir = tmp(), notices: string[] = []; + const { calls, fetchImpl } = capture(); + const ping = { event: "ai-scan" as const, probes: 12, findings: 2, severity: { high: 2 }, duration_ms: 40 }; + await sendUsagePing(ping, { fetchImpl, dir, env: {}, notice: (s) => notices.push(s) }); + await sendUsagePing(ping, { fetchImpl, dir, env: {}, notice: (s) => notices.push(s) }); + expect(notices).toHaveLength(1); + expect(notices[0]).toMatch(/THREATCRUSH_TELEMETRY=0/); + expect(calls).toHaveLength(2); + expect(calls[0].url).toMatch(/\/api\/telemetry$/); + expect(calls[0].body.install_id).toBe(calls[1].body.install_id); + expect(calls[0].body.install_id).toBe(JSON.parse(readFileSync(join(dir, "usage-ping.json"), "utf-8")).install_id); + }); + + it("carries counts and environment, never the target", async () => { + const { calls, fetchImpl } = capture(); + await sendUsagePing({ event: "ai-scan", probes: 3, findings: 0, duration_ms: 9 }, { fetchImpl, dir: tmp(), env: { CI: "true" }, notice: () => {} }); + const b = calls[0].body; + expect(Object.keys(b).sort()).toEqual(["arch", "ci", "duration_ms", "event", "findings", "install_id", "node", "platform", "probes", "version"]); + expect(b.ci).toBe(true); + }); + + it("never throws when the network fails", async () => { + const fetchImpl = (async () => { throw new Error("offline"); }) as unknown as typeof fetch; + await expect(sendUsagePing({ event: "ai-scan-run", scans: 1, findings: 0, duration_ms: 1 }, { fetchImpl, dir: tmp(), env: {}, notice: () => {} })).resolves.toBe(false); + }); + + it("reuses an id already on disk", () => { + const dir = tmp(); + const a = installId(dir), b = installId(dir); + expect(a.fresh).toBe(true); + expect(b).toEqual({ id: a.id, fresh: false }); + }); +}); diff --git a/apps/cli/src/core/usage-ping.ts b/apps/cli/src/core/usage-ping.ts new file mode 100644 index 00000000..84bae0e7 --- /dev/null +++ b/apps/cli/src/core/usage-ping.ts @@ -0,0 +1,90 @@ +import { randomUUID } from 'node:crypto'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { apiUrl, CLI_CONFIG_DIR } from './cli-config.js'; +import type { Severity } from './run-result.js'; +import { PKG_VERSION } from './version.js'; + +// One anonymous usage ping per ai-scan run, so threatcrush.com can count how +// often the scanner is used. ai-scan runs on the user's machine and otherwise +// sends nothing home. (Error reporting is a separate thing: core/telemetry.ts.) +// +// What is sent: the command, CLI version, OS, Node version, whether it ran in +// CI, a random per-machine install id, and counts (probes, findings by +// severity, scans in a .mosh script, duration). What is never sent: the target +// URL, headers, prompts, responses or finding text. +// +// Off with THREATCRUSH_TELEMETRY=0 (or off/false/no) or DO_NOT_TRACK=1. The +// first run prints a one-line notice saying so. + +const STATE_FILE = 'usage-ping.json'; + +export interface UsagePing { + event: 'ai-scan' | 'ai-scan-run'; + probes?: number; + scans?: number; + findings: number; + severity?: Partial>; + duration_ms: number; + failed?: boolean; +} + +export function usagePingDisabled(env: NodeJS.ProcessEnv = process.env): boolean { + const v = (env.THREATCRUSH_TELEMETRY ?? '').trim().toLowerCase(); + if (['0', 'off', 'false', 'no'].includes(v)) return true; + const dnt = (env.DO_NOT_TRACK ?? '').trim().toLowerCase(); + return dnt !== '' && dnt !== '0' && dnt !== 'false'; +} + +/** The random install id, created on first use; `fresh` is true that once (for the notice). */ +export function installId(dir = CLI_CONFIG_DIR): { id: string; fresh: boolean } { + const file = join(dir, STATE_FILE); + try { + const j = JSON.parse(readFileSync(file, 'utf-8')) as { install_id?: string }; + if (j.install_id) return { id: j.install_id, fresh: false }; + } catch {} + const id = randomUUID(); + try { + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + writeFileSync(file, JSON.stringify({ install_id: id, created_at: new Date().toISOString() }) + '\n'); + } catch {} + return { id, fresh: true }; +} + +export function pingBody(p: UsagePing, id: string, env: NodeJS.ProcessEnv = process.env) { + return { + ...p, + version: PKG_VERSION, + platform: process.platform, + arch: process.arch, + node: process.version, + ci: Boolean(env.CI || env.GITHUB_ACTIONS), + install_id: id, + }; +} + +/** Best effort: never throws, gives up after `timeoutMs` so it cannot hold the CLI open. */ +export async function sendUsagePing( + p: UsagePing, + opts: { fetchImpl?: typeof fetch; dir?: string; timeoutMs?: number; notice?: (s: string) => void; env?: NodeJS.ProcessEnv } = {}, +): Promise { + const env = opts.env ?? process.env; + if (usagePingDisabled(env)) return false; + try { + const { id, fresh } = installId(opts.dir); + if (fresh) { + (opts.notice ?? ((s) => process.stderr.write(s + '\n')))( + 'threatcrush: sends an anonymous usage count per ai-scan run (no targets, no findings text). Turn off with THREATCRUSH_TELEMETRY=0.', + ); + } + const res = await (opts.fetchImpl ?? fetch)(`${apiUrl()}/api/telemetry`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(pingBody(p, id, env)), + signal: AbortSignal.timeout(opts.timeoutMs ?? 2_000), + }); + return res.ok; + } catch { + return false; + } +} diff --git a/apps/web/src/app/api/telemetry/__tests__/route.test.ts b/apps/web/src/app/api/telemetry/__tests__/route.test.ts new file mode 100644 index 00000000..6f037cc9 --- /dev/null +++ b/apps/web/src/app/api/telemetry/__tests__/route.test.ts @@ -0,0 +1,64 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const inserted: unknown[] = []; +let insertError: { message: string } | null = null; + +vi.mock("@/lib/supabase", () => ({ + getSupabaseAdmin: () => ({ + from: (table: string) => ({ + insert: async (row: unknown) => { + inserted.push({ table, row }); + return { error: insertError }; + }, + }), + }), +})); + +import { POST } from "@/app/api/telemetry/route"; +import { parsePing } from "@/lib/usage-ping"; + +const req = (body: string) => + new Request("http://localhost/api/telemetry", { method: "POST", body }) as unknown as import("next/server").NextRequest; + +const PING = { + event: "ai-scan", probes: 12, findings: 2, severity: { high: 2, bogus: 9 }, duration_ms: 812, + version: "0.13.53", platform: "linux", arch: "x64", node: "v24.22.1", ci: false, + install_id: "0b9c3f2e-6a1d-4c55-9f3e-2d4b8a7c1e90", +}; + +describe("POST /api/telemetry", () => { + beforeEach(() => { inserted.length = 0; insertError = null; }); + + it("stores a valid ping and keeps only documented fields", async () => { + const res = await POST(req(JSON.stringify({ ...PING, target: "https://secret.example/llm", ip: "1.2.3.4" }))); + expect(res.status).toBe(202); + expect(inserted).toHaveLength(1); + const { table, row } = inserted[0] as { table: string; row: Record }; + expect(table).toBe("cli_usage_pings"); + expect(row).not.toHaveProperty("target"); + expect(row).not.toHaveProperty("ip"); + expect(row.severity).toEqual({ high: 2 }); + expect(row.findings).toBe(2); + }); + + it("rejects unknown events, junk and oversize bodies without storing", async () => { + for (const body of [JSON.stringify({ ...PING, event: "scan" }), "not json", "[]", "x".repeat(5000), ""]) { + expect((await POST(req(body))).status).toBe(400); + } + expect(inserted).toHaveLength(0); + }); + + it("drops malformed strings and clamps numbers", () => { + const row = parsePing({ ...PING, version: "