Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/middlewares/auto-moderation-stack/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ export const BANNED_DOMAINS = [
"amazon.it/gp/student",
"amazon.com/gp/student",
"polinetwork.it",
"mateapp.it",
Comment thread
toto04 marked this conversation as resolved.
Resolved
] as const

export const MULTI_CHAT_SPAM = {
Expand Down
3 changes: 2 additions & 1 deletion src/middlewares/auto-moderation-stack/functions.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { serviceBackend } from "@/backend"
import { BANNED_DOMAINS, POLINETWORK_DISCORD_GUILD_ID } from "./constants"
import { containsBannedDomain } from "./hostnames"

/**
* checks an array of strings for domains which are not allowed in our groups
Expand All @@ -10,7 +11,7 @@ import { BANNED_DOMAINS, POLINETWORK_DISCORD_GUILD_ID } from "./constants"
export async function checkForAllowedLinks(links: string[]): Promise<boolean> {
for (const url of links) {
// these websites are simply not allowed
if (BANNED_DOMAINS.some((domain) => url.includes(domain))) return false
if (containsBannedDomain(url, BANNED_DOMAINS)) return false

// specific discord invites handling
const discordMatches = url.matchAll(/discord\.gg\/([^/?#]+)/g)
Expand Down
48 changes: 48 additions & 0 deletions src/middlewares/auto-moderation-stack/hostnames.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
// scheme (optional) followed by a dotted hostname; also matches scheme-less links like `bit.ly/abc`
const HOSTNAME_REGEX = /(?:[a-z][a-z0-9+.-]*:\/\/)?((?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9-]*[a-z0-9])/gi

function normalizeHostname(host: string): string {
return host.toLowerCase().replace(/^www\./, "")
}

/**
* Extracts normalized (lowercase) hostnames from a string that is either a URL
* (e.g. a Telegram `text_link` entity) or free text containing URLs.
*/
export function extractHostnames(input: string): string[] {
const hosts = new Set<string>()

// well-formed absolute URL, e.g. from entities
try {
const parsed = new URL(input)
if (parsed.hostname) hosts.add(normalizeHostname(parsed.hostname.replace(/\.$/, "")))
} catch {
// not a standalone URL, fall through to text scanning
}

for (const match of input.matchAll(HOSTNAME_REGEX)) hosts.add(normalizeHostname(match[1]))
return [...hosts]
}

/**
* A hostname matches a banned domain if it is the domain itself or a subdomain of it
* (dot-delimited suffix), so `mateapp.it` and `www.mateapp.it` match but `climateapp.it` does not.
*/
export function hostMatchesDomain(host: string, domain: string): boolean {
return host === domain || host.endsWith(`.${domain}`)
}

/**
* Whether `input` contains a link to one of the banned entries.
* Entries without a path are matched against hostnames; entries with a path
* (e.g. `amazon.it/gp/student`) are matched against the lowercased input.
*/
export function containsBannedDomain(input: string, banned: readonly string[]): boolean {
const lowered = input.toLowerCase()
const hosts = extractHostnames(input)
return banned.some((entry) => {
const domain = entry.toLowerCase()
if (domain.includes("/")) return lowered.includes(domain)
return hosts.some((host) => hostMatchesDomain(host, domain))
})
}
40 changes: 40 additions & 0 deletions tests/banned-domains.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import { describe, expect, it } from "vitest"
import { BANNED_DOMAINS } from "@/middlewares/auto-moderation-stack/constants"
import { containsBannedDomain, extractHostnames } from "@/middlewares/auto-moderation-stack/hostnames"

const banned = (s: string) => containsBannedDomain(s, BANNED_DOMAINS)

describe("banned domains", () => {
it("extracts hostnames from entity URLs and raw text", () => {
expect(extractHostnames("https://WWW.MateApp.it/path?x=1")).toContain("mateapp.it")
expect(extractHostnames("check out bit.ly/abc and https://foo.example.com:8080/x")).toEqual(
expect.arrayContaining(["bit.ly", "foo.example.com"])
)
})

it.each([
"https://mateapp.it",
"https://www.mateapp.it/invite",
"https://sub.mateapp.it",
"HTTPS://MATEAPP.IT",
"mateapp.it",
"scarica mateapp.it/ref?abc oggi",
"https://mateapp.it.",
"https://mateapp.it:443/x",
"https://evil.com@mateapp.it",
"https://amazon.it/gp/student/x",
])("bans %s", (text) => {
expect(banned(text)).toBe(true)
})

it.each([
"https://climateapp.it",
"https://mateapp.it.evil.com",
"https://notmateapp.it",
"https://orbit.ly",
"https://example.com/mateapp.it-review",
"hello world",
])("does not ban %s", (text) => {
expect(banned(text)).toBe(false)
})
})
Loading