diff --git a/src/modules/post-regulation.ts b/src/modules/post-regulation.ts new file mode 100644 index 0000000..53f9c4f --- /dev/null +++ b/src/modules/post-regulation.ts @@ -0,0 +1,193 @@ +import { type Api, GrammyError, InputFile } from "grammy" +import type { InputMediaDocument, Message } from "grammy/types" +import { z } from "zod/v4" +import { wait } from "@/utils/wait" + +/** Telegram's limits for bot uploads and captions, and for documents in one media group. */ +const MAX_FILE_BYTES = 50 * 1024 * 1024 +const MAX_CAPTION_LENGTH = 1024 +const MAX_FILES = 10 + +/** The backend's `PostRegulation` (`src/websocket/telegram.ts`, PoliNetworkOrg/backend#117). */ +export type PostRegulationPayload = { + chatId: number + /** Forum topic (`message_thread_id`), or null for a chat without topics. */ + topicId: number | null + /** Telegram HTML, escaped by the backend. */ + text: string + /** The fields behind `text`; the bot posts `text` as is. */ + regulation: { + documentTitle: string + draftTitle: string + body: string + deliberaDate: string + oppositionDeadline: string | null + tag: string + /** The draft's diff in the dashboard, for people with dashboard access; null when not configured. */ + dashboardUrl: string | null + } + /** + * In order: the new official PDF, then the differences PDF. Base64 rather than bytes: the backend's Bun engine does + * not deliver socket.io binary attachments. + */ + files: { filename: string; contentType: string; base64: string }[] + /** + * The backend's notification id, the same on every retry. Same key, same post: a repeated command gets the first + * one's answer instead of posting again, whatever else it carries. + */ + idempotencyKey: string +} + +/** The `postRegulation` command; to be dropped once a published `@polinetwork/backend` exports it. */ +export type PostRegulationEvents = { + postRegulation: (data: PostRegulationPayload, cb: (error: string | null, result?: PostedMessages) => void) => void +} + +/** The messages posted; on failure, the ones posted before it (a partial post), possibly none. */ +export type PostedMessages = { messageIds: number[] } +export type PostRegulationResult = { error: string | null } & PostedMessages + +/** Only what the bot uses is checked; `regulation` is the backend's business. */ +const payloadSchema = z.object({ + chatId: z.number().int(), + topicId: z.number().int().positive().nullable(), + text: z.string().trim().min(1), + files: z + .array( + z + .object({ + filename: z.string().trim().min(1), + base64: z + .base64() + .transform((base64) => Buffer.from(base64, "base64")) + .refine((data) => data.byteLength > 0, "file is empty") + .refine((data) => data.byteLength <= MAX_FILE_BYTES, "file exceeds Telegram's 50 MB upload limit"), + }) + .transform(({ filename, base64 }) => ({ filename, data: base64 })) + ) + .min(1) + .max(MAX_FILES), + idempotencyKey: z.string().min(1), +}) + +export type RateLimitRetryOptions = { + /** Total attempts, the first included. Default 5. */ + maxAttempts?: number + /** + * Give up instead of waiting once the waits would add up to more than this. Default 30 s, so that with the uploads + * the bot answers within the backend's 60 s ack timeout, after which the backend sends the command again. + */ + maxTotalWaitMs?: number + sleep?: (ms: number) => Promise +} + +/** + * Runs `call`, waiting out Telegram's `retry_after` on 429 (Too Many Requests) and trying again. + * Only 429 is retried: Telegram did not perform the request, so trying again cannot post twice. + */ +export async function retryOnRateLimit(call: () => Promise, options: RateLimitRetryOptions = {}): Promise { + const { maxAttempts = 5, maxTotalWaitMs = 30_000, sleep = wait } = options + let waited = 0 + for (let attempt = 1; ; attempt++) { + try { + return await call() + } catch (error) { + if (!(error instanceof GrammyError) || error.error_code !== 429 || attempt >= maxAttempts) throw error + const delay = (error.parameters.retry_after ?? 1) * 1000 + if (waited + delay > maxTotalWaitMs) throw error + waited += delay + await sleep(delay) + } + } +} + +/** The length Telegram checks against its caption limit: the text once the HTML is parsed. */ +function visibleLength(html: string): number { + return html.replace(/<[^>]*>/g, "").replace(/&(lt|gt|quot|amp|#\d+|#x[\da-f]+);/gi, "_").length +} + +function describeError(error: unknown): string { + if (error instanceof GrammyError) return `${error.method}: ${error.error_code} ${error.description}` + if (error instanceof z.ZodError) return `invalid payload: ${z.prettifyError(error)}` + return error instanceof Error ? error.message : String(error) +} + +type PostApi = Pick + +/** + * Posts a regulation announcement with its documents: one media group (or one document) with the text as caption, + * or the text as a message followed by the documents when it does not fit in a caption. + */ +export async function postRegulation( + api: PostApi, + payload: unknown, + retry: RateLimitRetryOptions = {} +): Promise { + const messageIds: number[] = [] + try { + const { chatId, topicId, text, files } = payloadSchema.parse(payload) + const thread = topicId === null ? {} : { message_thread_id: topicId } + const caption = visibleLength(text) <= MAX_CAPTION_LENGTH ? text : null + const send = (call: () => Promise) => retryOnRateLimit(call, retry) + const record = (...messages: Message[]) => messageIds.push(...messages.map((m) => m.message_id)) + + if (caption === null) { + record( + await send(() => + api.sendMessage(chatId, text, { ...thread, parse_mode: "HTML", link_preview_options: { is_disabled: true } }) + ) + ) + } + + const documents = files.map((file) => new InputFile(file.data, file.filename)) + if (documents.length === 1) { + record( + await send(() => + api.sendDocument(chatId, documents[0], { + ...thread, + ...(caption === null ? {} : { caption, parse_mode: "HTML" }), + }) + ) + ) + } else { + // In a group of documents each one keeps its own caption, shown under it: on the last one it reads as the + // album's, on the first it would sit between the documents. + const media = documents.map( + (media, i): InputMediaDocument => + i === documents.length - 1 && caption !== null + ? { type: "document", media, caption, parse_mode: "HTML" } + : { type: "document", media } + ) + record(...(await send(() => api.sendMediaGroup(chatId, media, thread)))) + } + + return { error: null, messageIds } + } catch (error) { + return { error: describeError(error), messageIds } + } +} + +/** + * Remembers posts by idempotency key: a command repeated while the first is in flight waits for it, one repeated + * after a success gets the same message ids. Failures are forgotten so that the backend's retry posts again. + * Kept in memory: a restart of the bot forgets the keys. + */ +export class IdempotentPosts { + private readonly posts = new Map>() + + constructor(private readonly ttlMs = 24 * 60 * 60 * 1000) {} + + run(key: string | undefined, post: () => Promise): Promise { + if (key === undefined) return post() + const known = this.posts.get(key) + if (known) return known + + const pending = post().then((result) => { + if (result.error === null) setTimeout(() => this.posts.delete(key), this.ttlMs).unref() + else this.posts.delete(key) + return result + }) + this.posts.set(key, pending) + return pending + } +} diff --git a/src/modules/websocket.ts b/src/modules/websocket.ts index c62a885..66e6c54 100644 --- a/src/modules/websocket.ts +++ b/src/modules/websocket.ts @@ -1,7 +1,7 @@ import { type TelegramSocket, WS_PATH } from "@polinetwork/backend" import * as parser from "@socket.io/devalue-parser" import type { User } from "grammy/types" -import { io } from "socket.io-client" +import { io, type Socket } from "socket.io-client" import { serviceTokens } from "@/backend" import { env } from "@/env" import { Module } from "@/lib/modules" @@ -9,6 +9,7 @@ import { logger } from "@/logger" import { MessageUserStorage } from "@/middlewares/message-user-storage" import { duration } from "@/utils/duration" import type { ModuleShared } from "@/utils/types" +import { IdempotentPosts, type PostRegulationEvents, postRegulation } from "./post-regulation" import type { TgLogger } from "./tg-logger" import type { GrantActor } from "./tg-logger/types" import { SocketTokenAuth } from "./websocket-auth" @@ -46,15 +47,22 @@ type SocketError = { type: string } +/** The backend's commands, plus the ones the bot handles before `@polinetwork/backend` exports them. */ +type BotSocket = + TelegramSocket extends Socket + ? Socket + : never + /** * WebSocket to handle from-backend communication * * @param bot - The telegram bot instance */ export class WebSocketClient extends Module { - private io: TelegramSocket + private io: BotSocket private lastErrorCode: string | null = null private tokenAuth = new SocketTokenAuth(serviceTokens, { logger }) + private regulationPosts = new IdempotentPosts() constructor() { super() @@ -175,6 +183,14 @@ export class WebSocketClient extends Module { + const key = typeof data?.idempotencyKey === "string" ? data.idempotencyKey : undefined + const { error, messageIds } = await this.regulationPosts.run(key, () => postRegulation(this.shared.api, data)) + if (error === null) logger.info({ chatId: data.chatId, messageIds }, "[WS] regulation posted") + else logger.error({ error, messageIds }, "[WS] cannot post the regulation") + cb(error, { messageIds }) + }) } override async start() { diff --git a/tests/post-regulation.test.ts b/tests/post-regulation.test.ts new file mode 100644 index 0000000..7c7f00d --- /dev/null +++ b/tests/post-regulation.test.ts @@ -0,0 +1,274 @@ +import { GrammyError, InputFile } from "grammy" +import type { InputMediaDocument } from "grammy/types" +import { afterEach, describe, expect, it, vi } from "vitest" +import { + IdempotentPosts, + type PostRegulationPayload, + type PostRegulationResult, + postRegulation, + retryOnRateLimit, +} from "@/modules/post-regulation" + +const text = [ + "📙 Raccolta dei Regolamenti Interni: nuova versione rri-v2026.10.08", + "Approvato dal Consiglio Direttivo il 08/10/2026.", + "Pareri contrari e ricorsi entro il 28/10/2026.", + "In allegato il testo ufficiale (RaccoltaRegolamentiInterni.pdf) e il documento delle differenze (differenze-rri-v2026.10.08.pdf), che non ha valore ufficiale.", + 'Per chi ha accesso alla dashboard: le modifiche.', +].join("\n") + +const b64 = (bytes: number[] | Uint8Array) => Buffer.from(bytes).toString("base64") + +const payload = (overrides: Partial = {}): PostRegulationPayload => ({ + chatId: -1001234, + topicId: 42, + text, + regulation: { + documentTitle: "Raccolta dei Regolamenti Interni", + draftTitle: "Riforma dei gruppi", + body: "Consiglio Direttivo", + deliberaDate: "2026-10-08", + oppositionDeadline: "2026-10-28", + tag: "rri-v2026.10.08", + dashboardUrl: "https://dashboard.example.test/dashboard/regulations/drafts/d1/diff", + }, + files: [ + { filename: "RaccoltaRegolamentiInterni.pdf", contentType: "application/pdf", base64: b64([1, 2, 3]) }, + { filename: "differenze-rri-v2026.10.08.pdf", contentType: "application/pdf", base64: b64([4, 5]) }, + ], + idempotencyKey: "notification-1", + ...overrides, +}) + +const pdf = (base64: string) => ({ filename: "a.pdf", contentType: "application/pdf", base64 }) + +const rateLimited = (method: string, retryAfter: number) => + new GrammyError( + "Too Many Requests", + { + ok: false, + error_code: 429, + description: `Too Many Requests: retry after ${retryAfter}`, + parameters: { retry_after: retryAfter }, + }, + method, + {} + ) + +function fakeApi() { + let nextId = 100 + return { + sendMessage: vi.fn(async () => ({ message_id: nextId++ })), + sendDocument: vi.fn(async () => ({ message_id: nextId++ })), + sendMediaGroup: vi.fn(async (_chat: number, media: unknown[], _other?: object) => + media.map(() => ({ message_id: nextId++ })) + ), + } +} + +/** The fake only returns the message ids the code reads. */ +const asApi = (api: ReturnType) => api as unknown as Parameters[0] + +describe("postRegulation", () => { + it("posts the documents as one media group in the topic, the text as caption of the last one", async () => { + const api = fakeApi() + const result = await postRegulation(asApi(api), payload()) + + expect(result).toEqual({ error: null, messageIds: [100, 101] }) + expect(api.sendMessage).not.toHaveBeenCalled() + expect(api.sendMediaGroup).toHaveBeenCalledOnce() + const [chatId, media, other] = api.sendMediaGroup.mock.calls[0] as unknown as [number, InputMediaDocument[], object] + expect(chatId).toBe(-1001234) + expect(other).toEqual({ message_thread_id: 42 }) + expect(media).toHaveLength(2) + expect(media[0]).toEqual({ type: "document", media: expect.any(InputFile) }) + expect((media[0].media as InputFile).filename).toBe("RaccoltaRegolamentiInterni.pdf") + expect(media[1]).toMatchObject({ type: "document", caption: text, parse_mode: "HTML" }) + expect((media[1].media as InputFile).filename).toBe("differenze-rri-v2026.10.08.pdf") + }) + + it("posts outside topics when no topic is given", async () => { + const api = fakeApi() + await postRegulation(asApi(api), payload({ topicId: null })) + expect(api.sendMediaGroup.mock.calls[0][2]).toEqual({}) + }) + + it("sends a single document with sendDocument, since a media group needs two", async () => { + const api = fakeApi() + const result = await postRegulation(asApi(api), payload({ files: [pdf(b64([1]))] })) + expect(result).toEqual({ error: null, messageIds: [100] }) + expect(api.sendMediaGroup).not.toHaveBeenCalled() + expect(api.sendDocument).toHaveBeenCalledWith(-1001234, expect.any(InputFile), { + message_thread_id: 42, + caption: text, + parse_mode: "HTML", + }) + }) + + it("uploads the bytes the base64 encodes", async () => { + const api = fakeApi() + const bytes = Uint8Array.from({ length: 1000 }, (_, i) => (i * 7) % 256) + await postRegulation(asApi(api), payload({ files: [pdf(b64(bytes))] })) + const file = api.sendDocument.mock.calls[0] as unknown as [number, InputFile] + expect(new Uint8Array((await file[1].toRaw()) as Uint8Array)).toEqual(bytes) + }) + + it("sends the text as its own message when it does not fit in a caption", async () => { + const api = fakeApi() + const long = `${"x".repeat(1100)}` + const result = await postRegulation(asApi(api), payload({ text: long })) + + expect(result).toEqual({ error: null, messageIds: [100, 101, 102] }) + expect(api.sendMessage).toHaveBeenCalledWith(-1001234, long, { + message_thread_id: 42, + parse_mode: "HTML", + link_preview_options: { is_disabled: true }, + }) + const media = api.sendMediaGroup.mock.calls[0][1] as InputMediaDocument[] + expect(media.every((m) => !("caption" in m))).toBe(true) + }) + + it("counts the caption limit on the visible text, not the HTML", async () => { + const api = fakeApi() + // 800 entities are 4000 characters of HTML but 800 of text. + await postRegulation(asApi(api), payload({ text: `${"&".repeat(800)}` })) + expect(api.sendMessage).not.toHaveBeenCalled() + }) + + it("waits out a rate limit and tries again", async () => { + const api = fakeApi() + api.sendMediaGroup.mockRejectedValueOnce(rateLimited("sendMediaGroup", 7)) + const sleep = vi.fn(async () => {}) + + const result = await postRegulation(asApi(api), payload(), { sleep }) + + expect(result).toEqual({ error: null, messageIds: [100, 101] }) + expect(sleep).toHaveBeenCalledWith(7000) + expect(api.sendMediaGroup).toHaveBeenCalledTimes(2) + }) + + it("reports the messages already posted when a later call fails", async () => { + const api = fakeApi() + api.sendMediaGroup.mockRejectedValueOnce( + new GrammyError( + "Bad Request", + { ok: false, error_code: 400, description: "Bad Request: file too big" }, + "sendMediaGroup", + {} + ) + ) + + const result = await postRegulation(asApi(api), payload({ text: "x".repeat(1100) })) + + expect(result).toEqual({ error: "sendMediaGroup: 400 Bad Request: file too big", messageIds: [100] }) + }) + + it.each([ + ["no files", { files: [] }], + ["more than ten files", { files: Array.from({ length: 11 }, () => pdf(b64([1]))) }], + ["an empty file", { files: [pdf("")] }], + ["a file that is not base64", { files: [pdf("not base64!")] }], + ["a file over 50 MB", { files: [pdf(Buffer.alloc(50 * 1024 * 1024 + 1).toString("base64"))] }], + ["a missing idempotency key", { idempotencyKey: "" }], + ["an empty text", { text: " " }], + ["a missing topic field", { topicId: undefined as unknown as null }], + ])("refuses %s without calling Telegram", async (_, overrides) => { + const api = fakeApi() + const result = await postRegulation(asApi(api), payload(overrides as Partial)) + + expect(result.messageIds).toEqual([]) + expect(result.error).toMatch(/^invalid payload/) + expect(api.sendMessage).not.toHaveBeenCalled() + expect(api.sendDocument).not.toHaveBeenCalled() + expect(api.sendMediaGroup).not.toHaveBeenCalled() + }) +}) + +describe("retryOnRateLimit", () => { + it("retries only 429 errors", async () => { + const call = vi.fn().mockRejectedValue(new Error("network down")) + await expect(retryOnRateLimit(call, { sleep: async () => {} })).rejects.toThrow("network down") + expect(call).toHaveBeenCalledOnce() + }) + + it("gives up after the attempts allowed", async () => { + const call = vi.fn().mockRejectedValue(rateLimited("sendMessage", 1)) + const sleep = vi.fn(async () => {}) + await expect(retryOnRateLimit(call, { maxAttempts: 3, sleep })).rejects.toBeInstanceOf(GrammyError) + expect(call).toHaveBeenCalledTimes(3) + expect(sleep).toHaveBeenCalledTimes(2) + }) + + it("does not wait past 30 s in total by default, inside the backend's 60 s ack timeout", async () => { + const call = vi + .fn() + .mockRejectedValueOnce(rateLimited("sendMessage", 20)) + .mockRejectedValue(rateLimited("sendMessage", 11)) + const sleep = vi.fn(async () => {}) + await expect(retryOnRateLimit(call, { sleep })).rejects.toBeInstanceOf(GrammyError) + expect(sleep).toHaveBeenCalledOnce() + expect(call).toHaveBeenCalledTimes(2) + }) +}) + +describe("IdempotentPosts", () => { + afterEach(() => { + vi.useRealTimers() + }) + + const ok: PostRegulationResult = { error: null, messageIds: [1, 2] } + const failed: PostRegulationResult = { error: "boom", messageIds: [] } + + it("lets a command repeated while the first is in flight wait for it", async () => { + const posts = new IdempotentPosts() + let finish: (r: PostRegulationResult) => void = () => {} + const post = vi.fn( + () => + new Promise((r) => { + finish = r + }) + ) + + const first = posts.run("k", post) + const second = posts.run("k", post) + finish(ok) + + expect(await first).toEqual(ok) + expect(await second).toEqual(ok) + expect(post).toHaveBeenCalledOnce() + }) + + it("answers a repeat after a success with the same messages, until the TTL", async () => { + vi.useFakeTimers() + const posts = new IdempotentPosts(1000) + const post = vi.fn(async () => ok) + + await posts.run("k", post) + expect(await posts.run("k", post)).toEqual(ok) + expect(post).toHaveBeenCalledOnce() + + vi.advanceTimersByTime(1000) + await posts.run("k", post) + expect(post).toHaveBeenCalledTimes(2) + }) + + it("forgets failures so that a retry posts again", async () => { + const posts = new IdempotentPosts() + const post = vi.fn().mockResolvedValueOnce(failed).mockResolvedValueOnce(ok) + + expect(await posts.run("k", post)).toEqual(failed) + expect(await posts.run("k", post)).toEqual(ok) + expect(post).toHaveBeenCalledTimes(2) + }) + + it("does not dedupe without a key or across keys", async () => { + const posts = new IdempotentPosts() + const post = vi.fn(async () => ok) + + await posts.run(undefined, post) + await posts.run(undefined, post) + await posts.run("a", post) + await posts.run("b", post) + expect(post).toHaveBeenCalledTimes(4) + }) +})