From d2c09748ec657834f5d9c81371e0dd7e74e0e10c Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 28 Oct 2026 09:38:45 +0000 Subject: [PATCH] feat(cli): send threatcrush/ user agent to our API Node's fetch says "node", so threatcrush.com could not tell CLI versions apart or tell the CLI from any other Node script. The CLI and daemon now wrap fetch once at startup: requests to the configured API host and *.threatcrush.com that set no User-Agent carry "threatcrush/ (node ; )". Third-party hosts keep Node's default. scripts threatcrush-nightly already parses this UA for CLI users by version. Co-Authored-By: Claude Opus 5.5 --- .../cli/src/core/__tests__/user-agent.test.ts | 41 ++++++++++++++++ apps/cli/src/core/user-agent.ts | 47 +++++++++++++++++++ apps/cli/src/daemon-entry.ts | 3 ++ apps/cli/src/index.ts | 3 ++ 4 files changed, 94 insertions(+) create mode 100644 apps/cli/src/core/__tests__/user-agent.test.ts create mode 100644 apps/cli/src/core/user-agent.ts diff --git a/apps/cli/src/core/__tests__/user-agent.test.ts b/apps/cli/src/core/__tests__/user-agent.test.ts new file mode 100644 index 00000000..e12a5534 --- /dev/null +++ b/apps/cli/src/core/__tests__/user-agent.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { installUserAgent, isOurHost, USER_AGENT } from "../user-agent.js"; +import { PKG_VERSION } from "../version.js"; + +function fakeFetch() { + const seen: { url: string; ua: string | null }[] = []; + const target = { + fetch: (async (input: Parameters[0], init?: RequestInit) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + seen.push({ url, ua: new Headers(init?.headers).get("user-agent") }); + return new Response("ok"); + }) as typeof fetch, + }; + return { target, seen }; +} + +describe("user agent", () => { + it("names the CLI and its version in a form the nightly report parses", () => { + expect(USER_AGENT.startsWith(`threatcrush/${PKG_VERSION} `)).toBe(true); + }); + + it("only matches our own hosts", () => { + expect(isOurHost("https://threatcrush.com/api/ingest", "https://threatcrush.com")).toBe(true); + expect(isOurHost("https://supabase.threatcrush.com/rest/v1/x", "https://threatcrush.com")).toBe(true); + expect(isOurHost("http://localhost:3000/api/ingest", "http://localhost:3000")).toBe(true); + expect(isOurHost("https://evilthreatcrush.com/", "https://threatcrush.com")).toBe(false); + expect(isOurHost("https://feodotracker.abuse.ch/x", "https://threatcrush.com")).toBe(false); + expect(isOurHost("not a url", "https://threatcrush.com")).toBe(false); + }); + + it("adds the agent to our API, keeps other headers, leaves third parties and explicit agents alone", async () => { + const { target, seen } = fakeFetch(); + installUserAgent(target); + installUserAgent(target); // idempotent: no double wrap + await target.fetch("https://threatcrush.com/api/ingest", { method: "POST", headers: { authorization: "Bearer x" } }); + await target.fetch(new URL("https://threatcrush.com/api/auth/me")); + await target.fetch("https://discord.com/api/webhooks/1"); + await target.fetch("https://threatcrush.com/api/x", { headers: { "User-Agent": "custom" } }); + expect(seen.map((s) => s.ua)).toEqual([USER_AGENT, USER_AGENT, null, "custom"]); + }); +}); diff --git a/apps/cli/src/core/user-agent.ts b/apps/cli/src/core/user-agent.ts new file mode 100644 index 00000000..d2f3b345 --- /dev/null +++ b/apps/cli/src/core/user-agent.ts @@ -0,0 +1,47 @@ +import { apiUrl } from './cli-config.js'; +import { PKG_VERSION } from './version.js'; + +// Node's fetch says "node" and nothing else, so threatcrush.com cannot tell a +// CLI on 0.13.40 from one on 0.13.51, or the CLI from any other Node script. +// The CLI and the daemon both call installUserAgent() once at startup; every +// request to our own API that does not set a User-Agent then carries +// `threatcrush/`. Third-party hosts (abuse.ch, Discord, PagerDuty, a +// scan target) are left alone: they get Node's default, as before. + +export const USER_AGENT = `threatcrush/${PKG_VERSION} (node ${process.version}; ${process.platform})`; + +/** True for the configured API host and threatcrush.com or any of its subdomains. */ +export function isOurHost(url: string, api = apiUrl()): boolean { + let host: string; + try { + host = new URL(url).hostname.toLowerCase(); + } catch { + return false; + } + let apiHost = ''; + try { + apiHost = new URL(api).hostname.toLowerCase(); + } catch {} + return host === apiHost || host === 'threatcrush.com' || host.endsWith('.threatcrush.com'); +} + +function urlOf(input: Parameters[0]): string { + if (typeof input === 'string') return input; + if (input instanceof URL) return input.href; + return input.url; +} + +/** Wraps `target.fetch` so requests to our API carry the CLI's user agent. Idempotent. */ +export function installUserAgent(target: { fetch: typeof fetch } = globalThis): void { + const base = target.fetch as typeof fetch & { __threatcrushUA?: true }; + if (!base || base.__threatcrushUA) return; + const wrapped = ((input: Parameters[0], init?: RequestInit) => { + if (!isOurHost(urlOf(input))) return base(input, init); + const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined)); + if (headers.has('user-agent')) return base(input, init); + headers.set('user-agent', USER_AGENT); + return base(input, { ...init, headers }); + }) as typeof fetch & { __threatcrushUA?: true }; + wrapped.__threatcrushUA = true; + target.fetch = wrapped; +} diff --git a/apps/cli/src/daemon-entry.ts b/apps/cli/src/daemon-entry.ts index e286e041..266c4a34 100644 --- a/apps/cli/src/daemon-entry.ts +++ b/apps/cli/src/daemon-entry.ts @@ -1,5 +1,8 @@ #!/usr/bin/env node import { runDaemon } from './daemon/index.js'; +import { installUserAgent } from './core/user-agent.js'; + +installUserAgent(); runDaemon().catch((err) => { console.error('threatcrushd failed to start:', err); diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index 2b337c60..1e217ca2 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -69,6 +69,9 @@ import { import { IpcClient } from "./core/ipc-client.js"; import { findRunningDaemon } from "./daemon/pidfile.js"; import { PKG_VERSION } from "./core/version.js"; +import { installUserAgent } from "./core/user-agent.js"; + +installUserAgent(); const LOGO = ` ${chalk.green(" ████████╗██╗ ██╗██████╗ ███████╗ █████╗ ████████╗")}