From b48371b56a2cc3858bab15cb1005961fac086856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGOAT0P=E2=80=9D?= Date: Mon, 7 Sep 2026 16:12:28 +0530 Subject: [PATCH] fix: bundle referrer maps in backend builds --- .../src/analytics/utils/referrers.map.json | 82 +++++++++++++++++++ .../src/analytics/utils/referrers.map.ts | 34 ++------ backend/apps/cloud/tsconfig.app.json | 1 + .../src/analytics/utils/referrers.map.json | 82 +++++++++++++++++++ .../src/analytics/utils/referrers.map.ts | 34 ++------ .../v2/__tests__/referrers.map.spec.ts | 30 +++++++ backend/apps/community/tsconfig.app.json | 3 +- 7 files changed, 207 insertions(+), 59 deletions(-) create mode 100644 backend/apps/cloud/src/analytics/utils/referrers.map.json create mode 100644 backend/apps/community/src/analytics/utils/referrers.map.json create mode 100644 backend/apps/community/src/analytics/v2/__tests__/referrers.map.spec.ts diff --git a/backend/apps/cloud/src/analytics/utils/referrers.map.json b/backend/apps/cloud/src/analytics/utils/referrers.map.json new file mode 100644 index 000000000..f26681c27 --- /dev/null +++ b/backend/apps/cloud/src/analytics/utils/referrers.map.json @@ -0,0 +1,82 @@ +[ + { "name": "Linktree", "patterns": ["linktr.ee"] }, + { "name": "Perplexity", "patterns": ["perplexity.ai"], "category": "ai" }, + { "name": "Wikipedia", "patterns": ["wikipedia.org"] }, + { + "name": "Yandex", + "category": "search", + "patterns": [ + "yandex.com", + "yandex.ru", + "yandex.kz", + "yandex.by", + "ya.ru", + "yandex.eu", + "yandex.fr" + ] + }, + { "name": "GitHub", "patterns": ["github.com"] }, + { + "name": "Google", + "category": "search", + "patterns": [ + "google.com", + "doubleclick.net", + "android-app://com.google.android.gm/" + ] + }, + { + "name": "Bing", + "category": "search", + "patterns": ["bing.com", "ntp.msn.com"] + }, + { + "name": "DuckDuckGo", + "category": "search", + "patterns": ["duckduckgo.com", "ddg.gg"] + }, + { + "name": "Reddit", + "patterns": ["reddit.com", "android-app://com.reddit.frontpage/"] + }, + { "name": "YouTube", "patterns": ["youtube.com", "youtu.be"] }, + { "name": "Stack Overflow", "patterns": ["stackoverflow.com"] }, + { "name": "Hacker News", "patterns": ["news.ycombinator.com"] }, + { "name": "Medium", "patterns": ["medium.com"] }, + { "name": "Twitter", "patterns": ["x.com", "twitter.com", "t.co"] }, + { "name": "Facebook", "patterns": ["facebook.com"] }, + { "name": "LinkedIn", "patterns": ["linkedin.com"] }, + { "name": "Product Hunt", "patterns": ["producthunt.com"] }, + { + "name": "Microsoft Teams", + "patterns": ["teams.microsoft.com", "statics.teams.cdn.office.net"] + }, + { "name": "ChatGPT", "category": "ai", "patterns": ["chatgpt.com"] }, + { "name": "Claude", "category": "ai", "patterns": ["claude.ai"] }, + { "name": "Telegram", "patterns": ["telegram.org", "t.me"] }, + { "name": "Sogou", "category": "search", "patterns": ["sogou.com"] }, + { + "name": "Brave Search", + "category": "search", + "patterns": ["brave.com", "search.brave.com"] + }, + { "name": "Ecosia", "category": "search", "patterns": ["ecosia.org"] }, + { "name": "Baidu", "category": "search", "patterns": ["baidu.com"] }, + { + "name": "Yahoo!", + "category": "search", + "patterns": ["yahoo.com", "yahoo.co.jp"] + }, + { "name": "Qwant", "category": "search", "patterns": ["www.qwant.com"] }, + { "name": "Startpage", "category": "search", "patterns": ["startpage.com"] }, + { "name": "Instagram", "patterns": ["instagram.com"] }, + { "name": "Discord", "patterns": ["discord.com", "discordapp.com"] }, + { "name": "Gemini", "category": "ai", "patterns": ["gemini.google.com"] }, + { + "name": "Copilot", + "category": "ai", + "patterns": ["copilot.microsoft.com"] + }, + { "name": "OpenAI", "category": "ai", "patterns": ["openai.com"] }, + { "name": "Kagi", "category": "ai", "patterns": ["kagi.com"] } +] diff --git a/backend/apps/cloud/src/analytics/utils/referrers.map.ts b/backend/apps/cloud/src/analytics/utils/referrers.map.ts index ac8d4b40b..3b056aa20 100644 --- a/backend/apps/cloud/src/analytics/utils/referrers.map.ts +++ b/backend/apps/cloud/src/analytics/utils/referrers.map.ts @@ -1,37 +1,13 @@ -import fs from 'fs' -import path from 'path' +import MAP from './referrers.map.json' type ReferrerJson = { name: string; patterns: string[] } -let cachedMap: ReferrerJson[] | null = null - -const loadMap = (): ReferrerJson[] => { - if (cachedMap) return cachedMap - const candidates = [ - // Dev mode - path.resolve(__dirname, '../../../../../web/app/referrers.map.json'), - // Compiled mode - path.resolve(__dirname, '../../../../web/app/referrers.map.json'), - ] - for (const p of candidates) { - try { - const raw = fs.readFileSync(p, 'utf8') - cachedMap = JSON.parse(raw) as ReferrerJson[] - return cachedMap - } catch { - // try next location - } - } - console.warn( - '[WARN] Failed to load referrers map from all candidate paths, using empty map', - ) - cachedMap = [] - return cachedMap -} +const REFERRER_MAP = MAP as ReferrerJson[] export const getDomainsForRefName = (name: string): string[] | null => { - const list = loadMap() - const found = list.find((g) => g.name.toLowerCase() === name.toLowerCase()) + const found = REFERRER_MAP.find( + (g) => g.name.toLowerCase() === name.toLowerCase(), + ) if (!found) return null return found.patterns } diff --git a/backend/apps/cloud/tsconfig.app.json b/backend/apps/cloud/tsconfig.app.json index 536502cbc..540103c28 100644 --- a/backend/apps/cloud/tsconfig.app.json +++ b/backend/apps/cloud/tsconfig.app.json @@ -14,6 +14,7 @@ "incremental": true, "skipLibCheck": true, "inlineSources": true, + "resolveJsonModule": true, "sourceRoot": "/" } diff --git a/backend/apps/community/src/analytics/utils/referrers.map.json b/backend/apps/community/src/analytics/utils/referrers.map.json new file mode 100644 index 000000000..f26681c27 --- /dev/null +++ b/backend/apps/community/src/analytics/utils/referrers.map.json @@ -0,0 +1,82 @@ +[ + { "name": "Linktree", "patterns": ["linktr.ee"] }, + { "name": "Perplexity", "patterns": ["perplexity.ai"], "category": "ai" }, + { "name": "Wikipedia", "patterns": ["wikipedia.org"] }, + { + "name": "Yandex", + "category": "search", + "patterns": [ + "yandex.com", + "yandex.ru", + "yandex.kz", + "yandex.by", + "ya.ru", + "yandex.eu", + "yandex.fr" + ] + }, + { "name": "GitHub", "patterns": ["github.com"] }, + { + "name": "Google", + "category": "search", + "patterns": [ + "google.com", + "doubleclick.net", + "android-app://com.google.android.gm/" + ] + }, + { + "name": "Bing", + "category": "search", + "patterns": ["bing.com", "ntp.msn.com"] + }, + { + "name": "DuckDuckGo", + "category": "search", + "patterns": ["duckduckgo.com", "ddg.gg"] + }, + { + "name": "Reddit", + "patterns": ["reddit.com", "android-app://com.reddit.frontpage/"] + }, + { "name": "YouTube", "patterns": ["youtube.com", "youtu.be"] }, + { "name": "Stack Overflow", "patterns": ["stackoverflow.com"] }, + { "name": "Hacker News", "patterns": ["news.ycombinator.com"] }, + { "name": "Medium", "patterns": ["medium.com"] }, + { "name": "Twitter", "patterns": ["x.com", "twitter.com", "t.co"] }, + { "name": "Facebook", "patterns": ["facebook.com"] }, + { "name": "LinkedIn", "patterns": ["linkedin.com"] }, + { "name": "Product Hunt", "patterns": ["producthunt.com"] }, + { + "name": "Microsoft Teams", + "patterns": ["teams.microsoft.com", "statics.teams.cdn.office.net"] + }, + { "name": "ChatGPT", "category": "ai", "patterns": ["chatgpt.com"] }, + { "name": "Claude", "category": "ai", "patterns": ["claude.ai"] }, + { "name": "Telegram", "patterns": ["telegram.org", "t.me"] }, + { "name": "Sogou", "category": "search", "patterns": ["sogou.com"] }, + { + "name": "Brave Search", + "category": "search", + "patterns": ["brave.com", "search.brave.com"] + }, + { "name": "Ecosia", "category": "search", "patterns": ["ecosia.org"] }, + { "name": "Baidu", "category": "search", "patterns": ["baidu.com"] }, + { + "name": "Yahoo!", + "category": "search", + "patterns": ["yahoo.com", "yahoo.co.jp"] + }, + { "name": "Qwant", "category": "search", "patterns": ["www.qwant.com"] }, + { "name": "Startpage", "category": "search", "patterns": ["startpage.com"] }, + { "name": "Instagram", "patterns": ["instagram.com"] }, + { "name": "Discord", "patterns": ["discord.com", "discordapp.com"] }, + { "name": "Gemini", "category": "ai", "patterns": ["gemini.google.com"] }, + { + "name": "Copilot", + "category": "ai", + "patterns": ["copilot.microsoft.com"] + }, + { "name": "OpenAI", "category": "ai", "patterns": ["openai.com"] }, + { "name": "Kagi", "category": "ai", "patterns": ["kagi.com"] } +] diff --git a/backend/apps/community/src/analytics/utils/referrers.map.ts b/backend/apps/community/src/analytics/utils/referrers.map.ts index ac8d4b40b..3b056aa20 100644 --- a/backend/apps/community/src/analytics/utils/referrers.map.ts +++ b/backend/apps/community/src/analytics/utils/referrers.map.ts @@ -1,37 +1,13 @@ -import fs from 'fs' -import path from 'path' +import MAP from './referrers.map.json' type ReferrerJson = { name: string; patterns: string[] } -let cachedMap: ReferrerJson[] | null = null - -const loadMap = (): ReferrerJson[] => { - if (cachedMap) return cachedMap - const candidates = [ - // Dev mode - path.resolve(__dirname, '../../../../../web/app/referrers.map.json'), - // Compiled mode - path.resolve(__dirname, '../../../../web/app/referrers.map.json'), - ] - for (const p of candidates) { - try { - const raw = fs.readFileSync(p, 'utf8') - cachedMap = JSON.parse(raw) as ReferrerJson[] - return cachedMap - } catch { - // try next location - } - } - console.warn( - '[WARN] Failed to load referrers map from all candidate paths, using empty map', - ) - cachedMap = [] - return cachedMap -} +const REFERRER_MAP = MAP as ReferrerJson[] export const getDomainsForRefName = (name: string): string[] | null => { - const list = loadMap() - const found = list.find((g) => g.name.toLowerCase() === name.toLowerCase()) + const found = REFERRER_MAP.find( + (g) => g.name.toLowerCase() === name.toLowerCase(), + ) if (!found) return null return found.patterns } diff --git a/backend/apps/community/src/analytics/v2/__tests__/referrers.map.spec.ts b/backend/apps/community/src/analytics/v2/__tests__/referrers.map.spec.ts new file mode 100644 index 000000000..45ee9e66e --- /dev/null +++ b/backend/apps/community/src/analytics/v2/__tests__/referrers.map.spec.ts @@ -0,0 +1,30 @@ +import fs from 'fs' +import path from 'path' + +import { getDomainsForRefName } from '../../utils/referrers.map' + +describe('referrers map', () => { + it('resolves canonical referrer names case-insensitively', () => { + const google = getDomainsForRefName('Google') + expect(google).toContain('google.com') + expect(getDomainsForRefName('google')).toEqual(google) + expect(getDomainsForRefName('hypermonkey.tech')).toBeNull() + }) + + it('keeps both backend copies in sync with the web map', () => { + const files = [ + path.resolve(__dirname, '../../utils/referrers.map.json'), + path.resolve( + __dirname, + '../../../../../cloud/src/analytics/utils/referrers.map.json', + ), + path.resolve( + __dirname, + '../../../../../../../web/app/referrers.map.json', + ), + ] + const maps = files.map((file) => fs.readFileSync(file, 'utf8')) + + expect(new Set(maps).size).toBe(1) + }) +}) diff --git a/backend/apps/community/tsconfig.app.json b/backend/apps/community/tsconfig.app.json index 951e710e2..048e3f8d7 100644 --- a/backend/apps/community/tsconfig.app.json +++ b/backend/apps/community/tsconfig.app.json @@ -12,6 +12,7 @@ "outDir": "./dist", "baseUrl": "./", "incremental": true, - "skipLibCheck": true + "skipLibCheck": true, + "resolveJsonModule": true } }