From c0221f7a60e00053fbb287107fda0b33e9f61984 Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:07:11 +0200 Subject: [PATCH 001/139] Add v9.9.0 WhatsApp protocol primitives --- src/postmaster/whatsapp_v990/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/postmaster/whatsapp_v990/__init__.py diff --git a/src/postmaster/whatsapp_v990/__init__.py b/src/postmaster/whatsapp_v990/__init__.py new file mode 100644 index 00000000..3d04a345 --- /dev/null +++ b/src/postmaster/whatsapp_v990/__init__.py @@ -0,0 +1,4 @@ +"""Postmaster v9.9 clean-room WhatsApp Web protocol package.""" + +from .handshake import * # noqa: F401,F403 +from .pairing import * # noqa: F401,F403 From 01790dba1791fe236d0756ea8399d248c9b434a2 Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:07:13 +0200 Subject: [PATCH 002/139] Add v9.9.0 WhatsApp protocol primitives --- src/postmaster/whatsapp_v990/receipts.py | 36 ++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/postmaster/whatsapp_v990/receipts.py diff --git a/src/postmaster/whatsapp_v990/receipts.py b/src/postmaster/whatsapp_v990/receipts.py new file mode 100644 index 00000000..4bde397c --- /dev/null +++ b/src/postmaster/whatsapp_v990/receipts.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +class ReceiptPolicyError(PermissionError): + pass + + +@dataclass(frozen=True, slots=True) +class ReceiptDecision: + emit: bool + receipt_type: str | None + reason: str + + +def receipt_for_event(*, event: str, outbound_action: bool = False, reply_to_message: bool = False) -> ReceiptDecision: + """Enforce Postmaster's asymmetric WhatsApp receipt contract. + + Merely displaying/reading/syncing a message never emits a read receipt. The only permitted + read receipt path is coupled to an explicit outbound send/reply flow. + """ + normalized = str(event or "").strip().lower() + if normalized in {"display", "read", "sync", "history", "inspect"}: + return ReceiptDecision(False, None, "local_read_is_private") + if normalized in {"send", "reply"}: + if outbound_action and (normalized == "reply" or reply_to_message): + return ReceiptDecision(True, "read", "coupled_to_explicit_outbound_reply") + return ReceiptDecision(False, None, "outbound_send_without_reply_does_not_imply_read") + return ReceiptDecision(False, None, "no_receipt_for_event") + + +def assert_can_emit_read_receipt(*, outbound_action: bool, reply_to_message: bool) -> None: + decision = receipt_for_event(event="reply", outbound_action=outbound_action, reply_to_message=reply_to_message) + if not decision.emit: + raise ReceiptPolicyError("WhatsApp read receipt is allowed only during an explicit outbound reply flow") From a4839895efe1a6278803256474be2e6c2bbe4200 Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:07:15 +0200 Subject: [PATCH 003/139] Add v9.9.0 WhatsApp protocol primitives --- src/postmaster/whatsapp_v990/pairing.py | 76 +++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 src/postmaster/whatsapp_v990/pairing.py diff --git a/src/postmaster/whatsapp_v990/pairing.py b/src/postmaster/whatsapp_v990/pairing.py new file mode 100644 index 00000000..ad1ab2e0 --- /dev/null +++ b/src/postmaster/whatsapp_v990/pairing.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import base64 +from dataclasses import dataclass + + +PAIRING_QR_PREFIX = "https://wa.me/settings/linked_devices#" + + +class PairingError(ValueError): + pass + + +_BROWSER_TYPES = { + "chrome": 1, + "edge": 2, + "firefox": 3, + "ie": 4, + "opera": 5, + "safari": 6, +} + + +def companion_web_client_type(os_name: str, browser_name: str) -> int: + os_name = str(os_name or "").strip().lower() + browser_name = str(browser_name or "").strip() + if browser_name.lower() == "desktop": + return 8 if os_name == "windows" else 7 + return _BROWSER_TYPES.get(browser_name.lower(), 9) + + +def _b64(raw: bytes) -> str: + return base64.b64encode(bytes(raw)).decode("ascii") + + +def build_pairing_qr_data( + ref: str, + noise_public_key: bytes, + identity_public_key: bytes, + adv_secret_key: bytes | str, + *, + os_name: str = "Linux", + browser_name: str = "Chrome", +) -> str: + ref = str(ref or "").strip() + if not ref: + raise PairingError("Pairing reference is required") + if len(noise_public_key) != 32 or len(identity_public_key) != 32: + raise PairingError("Pairing public keys must be 32 bytes") + adv = adv_secret_key if isinstance(adv_secret_key, str) else _b64(adv_secret_key) + if not str(adv).strip(): + raise PairingError("ADV secret is required for companion QR") + fields = [ + ref, + _b64(noise_public_key), + _b64(identity_public_key), + str(adv), + str(companion_web_client_type(os_name, browser_name)), + ] + return PAIRING_QR_PREFIX + ",".join(fields) + + +@dataclass(frozen=True, slots=True) +class PairingQR: + payload: str + expires_at: str | None = None + + def public_status(self) -> dict[str, object]: + # Payload is intentionally omitted from generic status; it belongs only to the explicit + # pairing response/WebGUI view and is persisted encrypted when cached. + return {"available": bool(self.payload), "expires_at": self.expires_at, "payload_exposed_in_status": False} + + +__all__ = [ + "PAIRING_QR_PREFIX", "PairingError", "PairingQR", "companion_web_client_type", "build_pairing_qr_data", +] From 0a9eb011bfd4ba6b5d2ecc88c0451d964be45911 Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:07:18 +0200 Subject: [PATCH 004/139] Add v9.9.0 WhatsApp protocol primitives --- .../whatsapp_v990/websocket_driver.py | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 src/postmaster/whatsapp_v990/websocket_driver.py diff --git a/src/postmaster/whatsapp_v990/websocket_driver.py b/src/postmaster/whatsapp_v990/websocket_driver.py new file mode 100644 index 00000000..f901460a --- /dev/null +++ b/src/postmaster/whatsapp_v990/websocket_driver.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Awaitable, Callable + + +DEFAULT_WA_WEBSOCKET_URL = "wss://web.whatsapp.com/ws/chat" +DEFAULT_WA_ORIGIN = "https://web.whatsapp.com" + + +class WebSocketDriverError(RuntimeError): + pass + + +@dataclass(frozen=True, slots=True) +class WebSocketDriverConfig: + url: str = DEFAULT_WA_WEBSOCKET_URL + origin: str = DEFAULT_WA_ORIGIN + open_timeout: float = 20.0 + close_timeout: float = 10.0 + max_size: int | None = None + + +async def open_whatsapp_websocket( + config: WebSocketDriverConfig | None = None, + *, + connect_impl: Callable[..., Awaitable[Any]] | None = None, +) -> Any: + """Open the raw WA WebSocket using a lazy/optional websockets dependency. + + This function deliberately stops at the transport boundary: Noise XX and WhatsApp binary + protocol validation happen above it. Tests inject connect_impl so no external network is used. + """ + cfg = config or WebSocketDriverConfig() + if not str(cfg.url).startswith("wss://"): + raise WebSocketDriverError("WhatsApp WebSocket URL must use wss://") + connector = connect_impl + if connector is None: + try: + from websockets.asyncio.client import connect as connector # type: ignore + except Exception as exc: + raise WebSocketDriverError("Python websockets driver is unavailable") from exc + try: + return await connector( + cfg.url, + origin=cfg.origin, + open_timeout=cfg.open_timeout, + close_timeout=cfg.close_timeout, + max_size=cfg.max_size, + ) + except Exception as exc: + raise WebSocketDriverError(f"WhatsApp WebSocket connection failed: {type(exc).__name__}: {exc}") from exc + + +def driver_status() -> dict[str, object]: + try: + import websockets # noqa: F401 + available = True + except Exception: + available = False + return { + "endpoint": DEFAULT_WA_WEBSOCKET_URL, + "origin": DEFAULT_WA_ORIGIN, + "driver_available": available, + "noise_required": True, + "protocol_authenticated": False, + } + + +__all__ = [ + "DEFAULT_WA_WEBSOCKET_URL", "DEFAULT_WA_ORIGIN", "WebSocketDriverError", "WebSocketDriverConfig", + "open_whatsapp_websocket", "driver_status", +] From cf33c6bbab42d81f69e5af5a88dd57a6a2e06567 Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:07:20 +0200 Subject: [PATCH 005/139] Add v9.9.0 WhatsApp protocol primitives --- src/postmaster/whatsapp_v990/proto.py | 80 +++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 src/postmaster/whatsapp_v990/proto.py diff --git a/src/postmaster/whatsapp_v990/proto.py b/src/postmaster/whatsapp_v990/proto.py new file mode 100644 index 00000000..506d7b31 --- /dev/null +++ b/src/postmaster/whatsapp_v990/proto.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Iterable + + +class ProtoError(ValueError): + pass + + +WIRE_VARINT = 0 +WIRE_64BIT = 1 +WIRE_LENGTH = 2 +WIRE_32BIT = 5 + + +def encode_varint(value: int) -> bytes: + value = int(value) + if value < 0: + value &= (1 << 64) - 1 + out = bytearray() + while value > 0x7F: + out.append((value & 0x7F) | 0x80); value >>= 7 + out.append(value) + return bytes(out) + + +def decode_varint(data: bytes, offset: int = 0) -> tuple[int, int]: + result = 0; shift = 0; pos = int(offset) + while True: + if pos >= len(data): raise ProtoError("Truncated protobuf varint") + byte = data[pos]; pos += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): return result, pos + shift += 7 + if shift >= 70: raise ProtoError("Oversized protobuf varint") + + +@dataclass(frozen=True, slots=True) +class ProtoField: + number: int + wire_type: int + value: int | bytes + + +def field_varint(number: int, value: int) -> bytes: + return encode_varint((int(number) << 3) | WIRE_VARINT) + encode_varint(value) + + +def field_bytes(number: int, value: bytes | bytearray | memoryview | str) -> bytes: + raw = value.encode("utf-8") if isinstance(value, str) else bytes(value) + return encode_varint((int(number) << 3) | WIRE_LENGTH) + encode_varint(len(raw)) + raw + + +def field_message(number: int, fields: Iterable[bytes]) -> bytes: + return field_bytes(number, b"".join(fields)) + + +def decode_fields(data: bytes) -> list[ProtoField]: + raw = bytes(data); pos = 0; fields: list[ProtoField] = [] + while pos < len(raw): + key, pos = decode_varint(raw, pos) + number, wire = key >> 3, key & 7 + if number <= 0: raise ProtoError("Invalid protobuf field number") + if wire == WIRE_VARINT: + value, pos = decode_varint(raw, pos) + elif wire == WIRE_LENGTH: + length, pos = decode_varint(raw, pos) + if pos + length > len(raw): raise ProtoError("Truncated protobuf length-delimited field") + value = raw[pos:pos+length]; pos += length + elif wire == WIRE_64BIT: + if pos + 8 > len(raw): raise ProtoError("Truncated protobuf fixed64 field") + value = raw[pos:pos+8]; pos += 8 + elif wire == WIRE_32BIT: + if pos + 4 > len(raw): raise ProtoError("Truncated protobuf fixed32 field") + value = raw[pos:pos+4]; pos += 4 + else: + raise ProtoError(f"Unsupported protobuf wire type {wire}") + fields.append(ProtoField(number, wire, value)) + return fields From 62dd8df6a23f9ea0b8dee3eefa4808217ca720ff Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:07:22 +0200 Subject: [PATCH 006/139] Add v9.9.0 WhatsApp protocol primitives --- src/postmaster/whatsapp_v990/jid.py | 114 ++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 src/postmaster/whatsapp_v990/jid.py diff --git a/src/postmaster/whatsapp_v990/jid.py b/src/postmaster/whatsapp_v990/jid.py new file mode 100644 index 00000000..a9c02309 --- /dev/null +++ b/src/postmaster/whatsapp_v990/jid.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +class JIDError(ValueError): + pass + + +_DOMAIN_TYPES = { + "s.whatsapp.net": 0, + "c.us": 0, + "lid": 1, + "hosted": 128, + "hosted.lid": 129, +} + + +@dataclass(frozen=True, slots=True) +class JID: + user: str + server: str + device: int | None = None + agent: int | None = None + domain_type: int = 0 + + def __post_init__(self) -> None: + if not self.server or "@" in self.server: + raise JIDError("JID server is required and must not contain @") + if self.device is not None and not 0 <= int(self.device) <= 255: + raise JIDError("JID device must fit one byte") + if self.agent is not None and int(self.agent) < 0: + raise JIDError("JID agent must be non-negative") + if not 0 <= int(self.domain_type) <= 255: + raise JIDError("JID domain_type must fit one byte") + + @property + def bare(self) -> str: + return f"{self.user}@{self.server}" + + @property + def is_group(self) -> bool: + return self.server == "g.us" + + @property + def is_broadcast(self) -> bool: + return self.server == "broadcast" + + @property + def is_newsletter(self) -> bool: + return self.server == "newsletter" + + @property + def is_lid(self) -> bool: + return self.server in {"lid", "hosted.lid"} + + def normalized_user(self) -> "JID": + server = "s.whatsapp.net" if self.server == "c.us" else self.server + return JID(self.user, server, domain_type=_DOMAIN_TYPES.get(server, self.domain_type)) + + def __str__(self) -> str: + left = self.user + if self.agent not in (None, 0): + left += f"_{int(self.agent)}" + if self.device not in (None, 0): + left += f":{int(self.device)}" + return f"{left}@{self.server}" + + +def parse_jid(value: str) -> JID: + raw = str(value or "").strip() + pos = raw.find("@") + if pos < 0: + raise JIDError("JID must contain @") + left, server = raw[:pos], raw[pos + 1 :] + if not server: + raise JIDError("JID server is empty") + + device: int | None = None + agent: int | None = None + if ":" in left: + left, device_raw = left.rsplit(":", 1) + if not device_raw.isdigit(): + raise JIDError("JID device must be numeric") + device = int(device_raw) + if "_" in left: + user, agent_raw = left.rsplit("_", 1) + if agent_raw.isdigit(): + left = user + agent = int(agent_raw) + + return JID( + user=left, + server=server, + device=device, + agent=agent, + domain_type=_DOMAIN_TYPES.get(server, agent or 0), + ) + + +def encode_jid(user: str | int | None, server: str, device: int | None = None, agent: int | None = None) -> str: + return str(JID("" if user is None else str(user), str(server), device=device, agent=agent, domain_type=_DOMAIN_TYPES.get(str(server), agent or 0))) + + +def same_user(a: str | JID, b: str | JID) -> bool: + aa = a if isinstance(a, JID) else parse_jid(a) + bb = b if isinstance(b, JID) else parse_jid(b) + return aa.user == bb.user + + +def transfer_device(source: str | JID, target: str | JID) -> JID: + src = source if isinstance(source, JID) else parse_jid(source) + dst = target if isinstance(target, JID) else parse_jid(target) + return JID(dst.user, dst.server, device=src.device, agent=dst.agent, domain_type=dst.domain_type) From 8948d71bb6c70829f7ff542e119eb2ee51bf3183 Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:08:28 +0200 Subject: [PATCH 007/139] Add v9.9.0 WhatsApp handshake session primitives --- src/postmaster/whatsapp_v990/handshake.py | 108 ++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 src/postmaster/whatsapp_v990/handshake.py diff --git a/src/postmaster/whatsapp_v990/handshake.py b/src/postmaster/whatsapp_v990/handshake.py new file mode 100644 index 00000000..42929ced --- /dev/null +++ b/src/postmaster/whatsapp_v990/handshake.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from .proto import ProtoError, decode_fields, field_bytes, field_message + + +class HandshakeError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class ClientHello: + ephemeral: bytes + static: bytes = b"" + payload: bytes = b"" + + +@dataclass(frozen=True, slots=True) +class ServerHello: + ephemeral: bytes + static: bytes = b"" + payload: bytes = b"" + + +@dataclass(frozen=True, slots=True) +class ClientFinish: + static: bytes + payload: bytes + + +@dataclass(frozen=True, slots=True) +class HandshakeMessage: + client_hello: ClientHello | None = None + server_hello: ServerHello | None = None + client_finish: ClientFinish | None = None + + +def _bytes_fields(raw: bytes) -> dict[int, bytes]: + out: dict[int, bytes] = {} + try: + fields = decode_fields(raw) + except ProtoError as exc: + raise HandshakeError(str(exc)) from exc + for field in fields: + if not isinstance(field.value, bytes): + raise HandshakeError(f"Handshake field {field.number} has non-bytes wire type") + out[field.number] = field.value + return out + + +def _encode_hello(value: ClientHello | ServerHello) -> bytes: + fields = [field_bytes(1, value.ephemeral)] + if value.static: + fields.append(field_bytes(2, value.static)) + if value.payload: + fields.append(field_bytes(3, value.payload)) + return b"".join(fields) + + +def encode_handshake(message: HandshakeMessage) -> bytes: + variants = [message.client_hello is not None, message.server_hello is not None, message.client_finish is not None] + if sum(variants) != 1: + raise HandshakeError("HandshakeMessage must contain exactly one handshake phase") + if message.client_hello is not None: + return field_message(2, [_encode_hello(message.client_hello)]) + if message.server_hello is not None: + return field_message(3, [_encode_hello(message.server_hello)]) + finish = message.client_finish + assert finish is not None + return field_message(4, [field_bytes(1, finish.static), field_bytes(2, finish.payload)]) + + +def decode_handshake(raw: bytes) -> HandshakeMessage: + top = _bytes_fields(bytes(raw)) + present = [number for number in (2, 3, 4) if number in top] + if len(present) != 1: + raise HandshakeError("HandshakeMessage must contain exactly one recognized phase") + number = present[0] + nested = _bytes_fields(top[number]) + if number in (2, 3): + ephemeral = nested.get(1, b"") + if len(ephemeral) != 32: + raise HandshakeError("Handshake ephemeral public key must be 32 bytes") + cls = ClientHello if number == 2 else ServerHello + hello = cls(ephemeral=ephemeral, static=nested.get(2, b""), payload=nested.get(3, b"")) + return HandshakeMessage(client_hello=hello) if number == 2 else HandshakeMessage(server_hello=hello) + if 1 not in nested or 2 not in nested: + raise HandshakeError("ClientFinish requires static and payload") + return HandshakeMessage(client_finish=ClientFinish(static=nested[1], payload=nested[2])) + + +def encode_client_hello(ephemeral_public: bytes) -> bytes: + if len(ephemeral_public) != 32: + raise HandshakeError("ClientHello ephemeral public key must be 32 bytes") + return encode_handshake(HandshakeMessage(client_hello=ClientHello(ephemeral=bytes(ephemeral_public)))) + + +def encode_client_finish(encrypted_static: bytes, encrypted_payload: bytes) -> bytes: + return encode_handshake( + HandshakeMessage(client_finish=ClientFinish(static=bytes(encrypted_static), payload=bytes(encrypted_payload))) + ) + + +__all__ = [ + "HandshakeError", "ClientHello", "ServerHello", "ClientFinish", "HandshakeMessage", + "encode_handshake", "decode_handshake", "encode_client_hello", "encode_client_finish", +] From c5af4a34a4659fde3c8d0dede27d41e1ef5a165f Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:08:30 +0200 Subject: [PATCH 008/139] Add v9.9.0 WhatsApp handshake session primitives --- src/postmaster/whatsapp_v990/transport.py | 110 ++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 src/postmaster/whatsapp_v990/transport.py diff --git a/src/postmaster/whatsapp_v990/transport.py b/src/postmaster/whatsapp_v990/transport.py new file mode 100644 index 00000000..37a34f69 --- /dev/null +++ b/src/postmaster/whatsapp_v990/transport.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +import random +import time +from typing import Awaitable, Callable, Protocol + + +class TransportError(RuntimeError): + pass + + +class WebSocketLike(Protocol): + async def send(self, data: bytes) -> object: ... + async def recv(self) -> bytes | str: ... + async def close(self) -> object: ... + + +Connector = Callable[[], Awaitable[WebSocketLike]] + + +@dataclass(slots=True) +class ReconnectPolicy: + base_seconds: float = 1.0 + max_seconds: float = 30.0 + factor: float = 1.7 + jitter: float = 0.2 + + def delay(self, attempt: int, *, rand: Callable[[], float] = random.random) -> float: + raw = min(self.max_seconds, self.base_seconds * (self.factor ** max(0, int(attempt)))) + spread = raw * max(0.0, self.jitter) + return max(0.0, raw - spread + (2 * spread * rand())) + + +class ReconnectingTransport: + """Async WebSocket lifecycle with injected connector and explicit stop semantics. + + Network/library choice stays outside this module, keeping the clean-room protocol core pure + Python and avoiding a hard requirements.txt dependency. + """ + + def __init__(self, connector: Connector, *, policy: ReconnectPolicy | None = None): + self.connector = connector + self.policy = policy or ReconnectPolicy() + self.socket: WebSocketLike | None = None + self.connected = False + self.connect_attempts = 0 + self.reconnects = 0 + self.last_error: str | None = None + self.last_connected_at: float | None = None + self._stop = False + self._lock = asyncio.Lock() + + async def connect(self) -> None: + async with self._lock: + if self.connected and self.socket is not None: return + self.connect_attempts += 1 + try: + self.socket = await self.connector() + self.connected = True + self.last_error = None + self.last_connected_at = time.time() + except Exception as exc: + self.connected = False; self.socket = None; self.last_error = f"{type(exc).__name__}: {exc}" + raise TransportError(self.last_error) from exc + + async def ensure_connected(self, *, max_attempts: int | None = None, sleep: Callable[[float], Awaitable[object]] = asyncio.sleep) -> None: + attempt = 0 + while not self._stop: + try: + await self.connect(); return + except TransportError: + if max_attempts is not None and attempt + 1 >= max_attempts: raise + await sleep(self.policy.delay(attempt)); attempt += 1; self.reconnects += 1 + raise TransportError("Transport stopped") + + async def send(self, payload: bytes) -> None: + if not self.connected or self.socket is None: raise TransportError("WhatsApp transport is not connected") + try: + await self.socket.send(bytes(payload)) + except Exception as exc: + self.connected = False; self.last_error = f"{type(exc).__name__}: {exc}" + raise TransportError(self.last_error) from exc + + async def recv(self) -> bytes: + if not self.connected or self.socket is None: raise TransportError("WhatsApp transport is not connected") + try: + value = await self.socket.recv() + except Exception as exc: + self.connected = False; self.last_error = f"{type(exc).__name__}: {exc}" + raise TransportError(self.last_error) from exc + return value.encode("utf-8") if isinstance(value, str) else bytes(value) + + async def close(self) -> None: + self._stop = True + sock, self.socket = self.socket, None + self.connected = False + if sock is not None: + await sock.close() + + def status(self) -> dict[str, object]: + return { + "connected": self.connected, + "connect_attempts": self.connect_attempts, + "reconnects": self.reconnects, + "last_error": self.last_error, + "last_connected_at": self.last_connected_at, + "stopped": self._stop, + } From 29849af889cd3fd6ba7433bd743fb60429211f13 Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:08:32 +0200 Subject: [PATCH 009/139] Add v9.9.0 WhatsApp handshake session primitives --- src/postmaster/whatsapp_v990/signal_keys.py | 132 ++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 src/postmaster/whatsapp_v990/signal_keys.py diff --git a/src/postmaster/whatsapp_v990/signal_keys.py b/src/postmaster/whatsapp_v990/signal_keys.py new file mode 100644 index 00000000..980213b2 --- /dev/null +++ b/src/postmaster/whatsapp_v990/signal_keys.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +"""Clean-room Signal-v3 key bundle and X3DH-compatible derivation helpers. + +These helpers match the Curve25519/XEdDSA bundle shape used by WhatsApp's current +libsignal path. They are protocol primitives, not a claim of live server interoperability. +""" + +from dataclasses import dataclass +import hmac +import hashlib +import os + +from .crypto import ( + CurveKeyPair, + curve_shared_key, + generate_curve_keypair, + hkdf_sha256, + signal_public_key, + xeddsa_sign, + xeddsa_verify, +) + + +class SignalKeyError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class SignedPreKey: + key_id: int + key_pair: CurveKeyPair + signature: bytes + + +@dataclass(frozen=True, slots=True) +class SignalPreKeyBundle: + registration_id: int + identity_key: bytes + signed_pre_key_id: int + signed_pre_key: bytes + signed_pre_key_signature: bytes + pre_key_id: int | None = None + pre_key: bytes | None = None + + +def generate_registration_id(random2: bytes | None = None) -> int: + raw = bytes(random2 if random2 is not None else os.urandom(2)) + if len(raw) != 2: + raise SignalKeyError("Signal registration-id entropy must be exactly two bytes") + return int.from_bytes(raw, "little") & 0x3FFF + + +def generate_signed_pre_key(identity: CurveKeyPair, key_id: int, *, pre_key: CurveKeyPair | None = None, random64: bytes | None = None) -> SignedPreKey: + if not 0 <= int(key_id) <= 0xFFFFFF: + raise SignalKeyError("WhatsApp signed-prekey id must fit 24 bits") + pair = pre_key or generate_curve_keypair() + signature = xeddsa_sign(identity.private, signal_public_key(pair.public), random64=random64) + return SignedPreKey(key_id=int(key_id), key_pair=pair, signature=signature) + + +def verify_signed_pre_key(identity_public: bytes, signed_pre_key_public: bytes, signature: bytes) -> bool: + return xeddsa_verify(identity_public, signal_public_key(signed_pre_key_public), signature) + + +def derive_x3dh_initiator( + *, + our_identity_private: bytes, + our_base_private: bytes, + their_identity_public: bytes, + their_signed_pre_key_public: bytes, + their_one_time_pre_key_public: bytes | None = None, +) -> tuple[bytes, bytes]: + """Derive the legacy Signal-v3 initial root/chain pair for the initiator.""" + pieces = [ + b"ÿ" * 32, + curve_shared_key(our_identity_private, _raw_pub(their_signed_pre_key_public)), + curve_shared_key(our_base_private, _raw_pub(their_identity_public)), + curve_shared_key(our_base_private, _raw_pub(their_signed_pre_key_public)), + ] + if their_one_time_pre_key_public is not None: + pieces.append(curve_shared_key(our_base_private, _raw_pub(their_one_time_pre_key_public))) + material = hkdf_sha256(b"".join(pieces), 64, salt=None, info=b"WhisperText") + return material[:32], material[32:] + + +def derive_x3dh_responder( + *, + our_identity_private: bytes, + our_signed_pre_key_private: bytes, + their_identity_public: bytes, + their_base_public: bytes, + our_one_time_pre_key_private: bytes | None = None, +) -> tuple[bytes, bytes]: + """Responder mirror of :func:`derive_x3dh_initiator`.""" + pieces = [ + b"ÿ" * 32, + curve_shared_key(our_signed_pre_key_private, _raw_pub(their_identity_public)), + curve_shared_key(our_identity_private, _raw_pub(their_base_public)), + curve_shared_key(our_signed_pre_key_private, _raw_pub(their_base_public)), + ] + if our_one_time_pre_key_private is not None: + pieces.append(curve_shared_key(our_one_time_pre_key_private, _raw_pub(their_base_public))) + material = hkdf_sha256(b"".join(pieces), 64, salt=None, info=b"WhisperText") + return material[:32], material[32:] + + +def chain_message_seed(chain_key: bytes) -> bytes: + if len(chain_key) != 32: + raise SignalKeyError("Signal chain key must be 32 bytes") + return hmac.new(bytes(chain_key), b"", hashlib.sha256).digest() + + +def next_chain_key(chain_key: bytes) -> bytes: + if len(chain_key) != 32: + raise SignalKeyError("Signal chain key must be 32 bytes") + return hmac.new(bytes(chain_key), b"", hashlib.sha256).digest() + + +def derive_message_keys(chain_key: bytes) -> tuple[bytes, bytes, bytes]: + seed = chain_message_seed(chain_key) + material = hkdf_sha256(seed, 80, salt=None, info=b"WhisperMessageKeys") + return material[:32], material[32:64], material[64:80] + + +def _raw_pub(public: bytes) -> bytes: + raw = bytes(public) + if len(raw) == 33 and raw[0] == 5: + raw = raw[1:] + if len(raw) != 32: + raise SignalKeyError("Signal Curve25519 public key must be 32 bytes or 0x05-prefixed 33 bytes") + return raw From a0dda6a1dd552d4d4764eb1a53c0f2ee8733d07b Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:08:34 +0200 Subject: [PATCH 010/139] Add v9.9.0 WhatsApp handshake session primitives --- src/postmaster/whatsapp_v990/store.py | Bin 0 -> 5236 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/postmaster/whatsapp_v990/store.py diff --git a/src/postmaster/whatsapp_v990/store.py b/src/postmaster/whatsapp_v990/store.py new file mode 100644 index 0000000000000000000000000000000000000000..f42625f3259da0ec29d559713e9929761bffbef9 GIT binary patch literal 5236 zcmb_gZExE~67FaHiaCBbkOjlK4RF1>P;!b)x4O!bE747YARyM{%3@4%8SXNUFPHz` znc=Hmk{U_v58^Io=Y5`;<)%~_gyE(#RVl*|lB`gr0g>m*2$QH>k477!iBxXndy^*D zElrdvo#Z#8=ExhZ^7dKj`cNUvc5vbnmo15Yn5#V2eWF5JmMzt7Q+aF>`ZWC}M<{&jNV!9v00a)vxHuDJ$-4C%ZonSU%B0|5Q>jx<4^$ubcs z+-+4VwMga95Q#KJb3nu~>ZU=yFVZAROrv}*GN}s@NuGnQkAf= zJ~U@Odbn@(Cn+mgxmJYZ6{5rldf9E9ly<$)NDOLqiH%3)9WwucmPCUbQkmOG>cBt7@c4&Q zC`F>>et}v$MKvsttkMRqB_6E}7{r+bnDV|ii8knu%ZAblabq3ME|zCL1|9XJ9oxa5 zkVty}*T-00KKoBNQ71qCcnS*9W{X>V+{In2#oc4_eIyG5%QYj-p)QOwEr0s0E1103 z`MLHa%Bo5%JeApxf9Qp5VvG{LQw2JbtZ-?y44D@{)_4SK-mLN zF~Z>CP0Xw}u?yDSwdH`&uthymdD{GXcO^m+N3qknvD;_!5de2BCt-E&RC$`@w|15w zOy)O-bw)1P^I?t6^rHRaoasalJjiTGO@6tUz#D}fLZqDHQH{)_OyO^wA!f*ad zyR+YASM=rFdU}|M=4Sn&O9x^|A+jYM42>O6|)~X@r8(cX-yX<>HK&M_?z;HT_UorXp{ zZVhsfr0u)|I42-NzpWA+$p#_GIrA8YTa5mGfYF0|8^eT0(WZi_^O5gpytJ8%Tib85 zGsr0J_Z&sG-{}*95utuhBZLcIfexLk+61&_i?zQBV73UB#|pcTfJ?D(Yvp@O-M(xb z_+v7^^4IoLxBfpU7zYk<##%05x?G&kXVYL$m;+}^xVk)>1U{_&03(WpKgKqZR#W+2HzSMw! zH|$q@OT#<&>mvZn7Wa7GL2%9A1rq4RRaR&_PH@)hAPm1S&=sYN$n`SDx>Dkr}`3qu88*%;J<4G$@U5)a!}WAaL64rrY7lG z#XdV?nM#}p$qApyjT(EDO8dP1O5lj9@&Gu_5I7LVzZfTWhk?NmFldyANK8BJU*jWG zjz;%L;m06&0w=w@Xt1g@yX#1X{e9kH$zw=zV;=?A>GEn3*xx#zbe1(l%U36_`lErQ8&>x<3P; Date: Mon, 21 Sep 2026 20:09:34 +0200 Subject: [PATCH 011/139] Add v9.9.0 WhatsApp cryptographic primitives --- src/postmaster/whatsapp_v990/crypto.py | 409 +++++++++++++++++++++++++ 1 file changed, 409 insertions(+) create mode 100644 src/postmaster/whatsapp_v990/crypto.py diff --git a/src/postmaster/whatsapp_v990/crypto.py b/src/postmaster/whatsapp_v990/crypto.py new file mode 100644 index 00000000..5cabdc00 --- /dev/null +++ b/src/postmaster/whatsapp_v990/crypto.py @@ -0,0 +1,409 @@ +from __future__ import annotations + +"""Clean-room WhatsApp/Signal cryptographic primitives for Postmaster v9.9. + +Only public protocol specifications are used here. The module intentionally keeps the +WhatsApp-specific policy thin: Curve25519/XEdDSA, Noise XX helpers, and media key derivation. +Long-lived secrets are never logged or returned by the public MCP surface. +""" + +from dataclasses import dataclass +import hashlib +import hmac +import os +from typing import Final + +from cryptography.hazmat.primitives import hashes, padding, serialization +from cryptography.hazmat.primitives.asymmetric import x25519 +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.kdf.hkdf import HKDF + +P: Final[int] = 2**255 - 19 +Q: Final[int] = 2**252 + 27742317777372353535851937790883648493 +D: Final[int] = (-121665 * pow(121666, P - 2, P)) % P +SQRT_M1: Final[int] = pow(2, (P - 1) // 4, P) +NOISE_MODE: Final[bytes] = b"Noise_XX_25519_AESGCM_SHA256\x00\x00\x00\x00" +NOISE_WA_HEADER: Final[bytes] = bytes((87, 65, 6, 3)) +WA_CERT_PUBLIC_KEY: Final[bytes] = bytes.fromhex("142375574d0a587166aae71ebe516437c4a28b73e3695c6ce1f7f9545da8ee6b") + + +class CryptoError(ValueError): + pass + + +@dataclass(frozen=True) +class CurveKeyPair: + private: bytes + public: bytes + + +def clamp_curve25519_scalar(raw: bytes) -> bytes: + if len(raw) != 32: + raise CryptoError("Curve25519 private key must be 32 bytes") + b = bytearray(raw) + b[0] &= 248 + b[31] &= 127 + b[31] |= 64 + return bytes(b) + + +def generate_curve_keypair() -> CurveKeyPair: + private = x25519.X25519PrivateKey.generate() + priv = private.private_bytes(serialization.Encoding.Raw, serialization.PrivateFormat.Raw, serialization.NoEncryption()) + pub = private.public_key().public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) + return CurveKeyPair(private=priv, public=pub) + + +def curve_public_from_private(private: bytes) -> bytes: + key = x25519.X25519PrivateKey.from_private_bytes(private) + return key.public_key().public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) + + +def curve_shared_key(private: bytes, public: bytes) -> bytes: + if len(private) != 32 or len(public) != 32: + raise CryptoError("Curve25519 keys must be 32 bytes") + return x25519.X25519PrivateKey.from_private_bytes(private).exchange(x25519.X25519PublicKey.from_public_bytes(public)) + + +def signal_public_key(public: bytes) -> bytes: + if len(public) == 33 and public[0] == 5: + return public + if len(public) != 32: + raise CryptoError("Signal Curve25519 public key must be 32 bytes") + return b"\x05" + public + + +Point = tuple[int, int] +IDENTITY: Final[Point] = (0, 1) + + +def _inv(x: int) -> int: + return pow(x % P, P - 2, P) if x % P else 0 + + +def _sqrt(v: int) -> int | None: + v %= P + x = pow(v, (P + 3) // 8, P) + if (x * x - v) % P: + x = (x * SQRT_M1) % P + if (x * x - v) % P: + return None + return x + + +def _point_from_y(y: int, sign: int) -> Point: + if not 0 <= y < P: + raise CryptoError("Invalid Edwards y coordinate") + y2 = y * y % P + x2 = (y2 - 1) * _inv(1 + D * y2) % P + x = _sqrt(x2) + if x is None: + raise CryptoError("Point is not on Curve25519 Edwards form") + if (x & 1) != (sign & 1): + x = (-x) % P + point = (x, y) + if not _on_curve(point): + raise CryptoError("Decoded point is not on curve") + return point + + +def _on_curve(point: Point) -> bool: + x, y = point + return (-x * x + y * y - 1 - D * x * x * y * y) % P == 0 + + +def _point_add(a: Point, b: Point) -> Point: + x1, y1 = a + x2, y2 = b + t = D * x1 * x2 * y1 * y2 % P + den_x = _inv(1 + t) + den_y = _inv(1 - t) + x3 = (x1 * y2 + y1 * x2) * den_x % P + y3 = (y1 * y2 + x1 * x2) * den_y % P + return x3, y3 + + +def _point_neg(point: Point) -> Point: + return (-point[0]) % P, point[1] + + +def _scalar_mult(k: int, point: Point) -> Point: + k %= Q + acc = IDENTITY + cur = point + while k: + if k & 1: + acc = _point_add(acc, cur) + cur = _point_add(cur, cur) + k >>= 1 + return acc + + +def _encode_point(point: Point) -> bytes: + x, y = point + raw = bytearray(int(y).to_bytes(32, "little")) + raw[31] = (raw[31] & 0x7F) | ((x & 1) << 7) + return bytes(raw) + + +def _decode_point(raw: bytes) -> Point: + if len(raw) != 32: + raise CryptoError("Compressed Edwards point must be 32 bytes") + value = int.from_bytes(raw, "little") + sign = value >> 255 + y = value & ((1 << 255) - 1) + return _point_from_y(y, sign) + + +def _convert_montgomery_u(public: bytes) -> Point: + if len(public) == 33 and public[0] == 5: + public = public[1:] + if len(public) != 32: + raise CryptoError("Montgomery public key must be 32 bytes") + u = int.from_bytes(public, "little") & ((1 << 255) - 1) + y = (u - 1) * _inv(u + 1) % P + return _point_from_y(y, 0) + + +BASE_POINT: Final[Point] = _convert_montgomery_u((9).to_bytes(32, "little")) + + +def _xeddsa_keypair(private: bytes) -> tuple[Point, int]: + scalar_bytes = clamp_curve25519_scalar(private) + k = int.from_bytes(scalar_bytes, "little") % Q + e = _scalar_mult(k, BASE_POINT) + a = (-k) % Q if (e[0] & 1) else k + A = (e[0] if not (e[0] & 1) else (-e[0]) % P, e[1]) + if A[0] & 1: + raise CryptoError("Failed to normalize XEdDSA public key") + return A, a + + +def _hash_int(data: bytes) -> int: + return int.from_bytes(hashlib.sha512(data).digest(), "little") + + +def _hash_i(index: int, data: bytes) -> int: + if not 0 <= index < 256: + raise CryptoError("XEdDSA domain index must fit one byte") + prefix = ((1 << 256) - 1 - index).to_bytes(32, "little") + return _hash_int(prefix + data) + + +def xeddsa_sign(private: bytes, message: bytes, *, random64: bytes | None = None) -> bytes: + if len(private) != 32: + raise CryptoError("XEdDSA private key must be 32 bytes") + z = random64 if random64 is not None else os.urandom(64) + if len(z) != 64: + raise CryptoError("XEdDSA randomizer must be 64 bytes") + A, a = _xeddsa_keypair(private) + a_bytes = a.to_bytes(32, "little") + r = _hash_i(1, a_bytes + bytes(message) + z) % Q + R = _scalar_mult(r, BASE_POINT) + Rb = _encode_point(R) + Ab = _encode_point(A) + h = _hash_int(Rb + Ab + bytes(message)) % Q + s = (r + h * a) % Q + return Rb + s.to_bytes(32, "little") + + +def xeddsa_public_edwards(public: bytes) -> bytes: + return _encode_point(_convert_montgomery_u(public)) + + +def xeddsa_verify(public: bytes, message: bytes, signature: bytes) -> bool: + try: + if len(signature) != 64: + return False + A = _convert_montgomery_u(public) + Rb = signature[:32] + R = _decode_point(Rb) + s = int.from_bytes(signature[32:], "little") + if s >= Q: + return False + Ab = _encode_point(A) + h = _hash_int(Rb + Ab + bytes(message)) % Q + check = _point_add(_scalar_mult(s, BASE_POINT), _point_neg(_scalar_mult(h, A))) + return hmac.compare_digest(_encode_point(check), Rb) + except (CryptoError, ValueError): + return False + + +def hkdf_sha256(ikm: bytes, length: int, *, salt: bytes | None = None, info: bytes = b"") -> bytes: + return HKDF(algorithm=hashes.SHA256(), length=int(length), salt=salt, info=info).derive(bytes(ikm)) + + +def aes_cbc_encrypt(key: bytes, iv: bytes, plaintext: bytes) -> bytes: + padder = padding.PKCS7(128).padder() + padded = padder.update(bytes(plaintext)) + padder.finalize() + enc = Cipher(algorithms.AES(bytes(key)), modes.CBC(bytes(iv))).encryptor() + return enc.update(padded) + enc.finalize() + + +def aes_cbc_decrypt(key: bytes, iv: bytes, ciphertext: bytes) -> bytes: + dec = Cipher(algorithms.AES(bytes(key)), modes.CBC(bytes(iv))).decryptor() + padded = dec.update(bytes(ciphertext)) + dec.finalize() + unpadder = padding.PKCS7(128).unpadder() + return unpadder.update(padded) + unpadder.finalize() + + +_MEDIA_INFO = { + "image": b"WhatsApp Image Keys", + "video": b"WhatsApp Video Keys", + "audio": b"WhatsApp Audio Keys", + "document": b"WhatsApp Document Keys", + "sticker": b"WhatsApp Image Keys", +} + + +def encrypt_media(plaintext: bytes, media_type: str, *, media_key: bytes | None = None) -> dict[str, bytes]: + kind = str(media_type).lower() + if kind not in _MEDIA_INFO: + raise CryptoError(f"Unsupported WhatsApp media type: {media_type}") + key = media_key or os.urandom(32) + if len(key) != 32: + raise CryptoError("WhatsApp media key must be 32 bytes") + expanded = hkdf_sha256(key, 112, salt=None, info=_MEDIA_INFO[kind]) + iv, cipher_key, mac_key = expanded[:16], expanded[16:48], expanded[48:80] + encrypted = aes_cbc_encrypt(cipher_key, iv, bytes(plaintext)) + mac = hmac.new(mac_key, iv + encrypted, hashlib.sha256).digest()[:10] + blob = encrypted + mac + return { + "media_key": key, + "file_enc_sha256": hashlib.sha256(blob).digest(), + "file_sha256": hashlib.sha256(bytes(plaintext)).digest(), + "encrypted": blob, + } + + +def decrypt_media(blob: bytes, media_type: str, media_key: bytes, *, expected_file_sha256: bytes | None = None) -> bytes: + kind = str(media_type).lower() + if kind not in _MEDIA_INFO: + raise CryptoError(f"Unsupported WhatsApp media type: {media_type}") + if len(blob) < 11 or len(media_key) != 32: + raise CryptoError("Invalid WhatsApp media payload") + expanded = hkdf_sha256(media_key, 112, salt=None, info=_MEDIA_INFO[kind]) + iv, cipher_key, mac_key = expanded[:16], expanded[16:48], expanded[48:80] + encrypted, mac = bytes(blob[:-10]), bytes(blob[-10:]) + expected = hmac.new(mac_key, iv + encrypted, hashlib.sha256).digest()[:10] + if not hmac.compare_digest(mac, expected): + raise CryptoError("WhatsApp media MAC mismatch") + plaintext = aes_cbc_decrypt(cipher_key, iv, encrypted) + if expected_file_sha256 is not None and not hmac.compare_digest(hashlib.sha256(plaintext).digest(), expected_file_sha256): + raise CryptoError("WhatsApp media SHA-256 mismatch") + return plaintext + + +def _noise_iv(counter: int) -> bytes: + if counter < 0 or counter > 0xFFFFFFFF: + raise CryptoError("Noise nonce counter exhausted") + return b"\x00" * 8 + int(counter).to_bytes(4, "big") + + +@dataclass +class NoiseTransport: + write_key: bytes + read_key: bytes + write_counter: int = 0 + read_counter: int = 0 + + def encrypt(self, plaintext: bytes) -> bytes: + iv = _noise_iv(self.write_counter) + self.write_counter += 1 + return AESGCM(self.write_key).encrypt(iv, bytes(plaintext), b"") + + def decrypt(self, ciphertext: bytes) -> bytes: + iv = _noise_iv(self.read_counter) + self.read_counter += 1 + try: + return AESGCM(self.read_key).decrypt(iv, bytes(ciphertext), b"") + except Exception as exc: + raise CryptoError("Noise transport authentication failed") from exc + + +class WhatsAppNoiseXX: + """Initiator-side Noise_XX_25519_AESGCM_SHA256 state used by WhatsApp Web.""" + + def __init__(self, ephemeral: CurveKeyPair, *, header: bytes = NOISE_WA_HEADER): + self.ephemeral = ephemeral + initial = hashlib.sha256(NOISE_MODE).digest() if len(NOISE_MODE) != 32 else NOISE_MODE + self.hash = initial + self.salt = initial + self.key = initial + self.counter = 0 + self.transport: NoiseTransport | None = None + self.authenticate(header) + self.authenticate(ephemeral.public) + + def authenticate(self, data: bytes) -> None: + if self.transport is None: + self.hash = hashlib.sha256(self.hash + bytes(data)).digest() + + def _derive_pair(self, data: bytes) -> tuple[bytes, bytes]: + material = hkdf_sha256(bytes(data), 64, salt=self.salt, info=b"") + return material[:32], material[32:] + + def mix_key(self, data: bytes) -> None: + self.salt, self.key = self._derive_pair(data) + self.counter = 0 + + def encrypt(self, plaintext: bytes) -> bytes: + if self.transport is not None: + return self.transport.encrypt(plaintext) + iv = _noise_iv(self.counter) + self.counter += 1 + out = AESGCM(self.key).encrypt(iv, bytes(plaintext), self.hash) + self.authenticate(out) + return out + + def decrypt(self, ciphertext: bytes) -> bytes: + if self.transport is not None: + return self.transport.decrypt(ciphertext) + iv = _noise_iv(self.counter) + self.counter += 1 + try: + out = AESGCM(self.key).decrypt(iv, bytes(ciphertext), self.hash) + except Exception as exc: + raise CryptoError("Noise handshake authentication failed") from exc + self.authenticate(ciphertext) + return out + + def process_server_hello(self, *, server_ephemeral: bytes, encrypted_static: bytes, encrypted_payload: bytes, + noise_static: CurveKeyPair) -> tuple[bytes, bytes]: + self.authenticate(server_ephemeral) + self.mix_key(curve_shared_key(self.ephemeral.private, server_ephemeral)) + server_static = self.decrypt(encrypted_static) + if len(server_static) != 32: + raise CryptoError("Noise server static key must be 32 bytes") + self.mix_key(curve_shared_key(self.ephemeral.private, server_static)) + cert_chain = self.decrypt(encrypted_payload) + encrypted_client_static = self.encrypt(noise_static.public) + self.mix_key(curve_shared_key(noise_static.private, server_ephemeral)) + return encrypted_client_static, cert_chain + + def finish(self) -> NoiseTransport: + write, read = self._derive_pair(b"") + self.transport = NoiseTransport(write_key=write, read_key=read) + return self.transport + + +def frame_noise_payload(payload: bytes, *, intro: bytes | None = None) -> bytes: + size = len(payload) + if size > 0xFFFFFF: + raise CryptoError("WhatsApp Noise frame exceeds 24-bit length") + return (bytes(intro or b"") + size.to_bytes(3, "big") + bytes(payload)) + + +def split_noise_frames(buffer: bytes) -> tuple[list[bytes], bytes]: + frames: list[bytes] = [] + pos = 0 + data = bytes(buffer) + while len(data) - pos >= 3: + size = int.from_bytes(data[pos:pos+3], "big") + if len(data) - pos - 3 < size: + break + pos += 3 + frames.append(data[pos:pos+size]) + pos += size + return frames, data[pos:] From 939e2b83e6ca7d9c37baf9f19fbb786586dbe6fc Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:10:28 +0200 Subject: [PATCH 012/139] Add v9.9.0 WhatsApp WABinary codec --- src/postmaster/whatsapp_v990/binary.py | 306 +++++++++++++++++++++++++ 1 file changed, 306 insertions(+) create mode 100644 src/postmaster/whatsapp_v990/binary.py diff --git a/src/postmaster/whatsapp_v990/binary.py b/src/postmaster/whatsapp_v990/binary.py new file mode 100644 index 00000000..86975eca --- /dev/null +++ b/src/postmaster/whatsapp_v990/binary.py @@ -0,0 +1,306 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +import zlib +from typing import Iterable, Mapping, Sequence + +from .jid import JID, parse_jid + + +class BinaryNodeError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class BinaryTags: + LIST_EMPTY: int = 0 + DICTIONARY_0: int = 236 + DICTIONARY_1: int = 237 + DICTIONARY_2: int = 238 + DICTIONARY_3: int = 239 + INTEROP_JID: int = 245 + FB_JID: int = 246 + AD_JID: int = 247 + LIST_8: int = 248 + LIST_16: int = 249 + JID_PAIR: int = 250 + HEX_8: int = 251 + BINARY_8: int = 252 + BINARY_20: int = 253 + BINARY_32: int = 254 + NIBBLE_8: int = 255 + PACKED_MAX: int = 127 + + +TAGS = BinaryTags() + + +@dataclass(slots=True) +class BinaryNode: + tag: str + attrs: dict[str, str] = field(default_factory=dict) + content: bytes | str | list["BinaryNode"] | None = None + + def child(self, tag: str) -> "BinaryNode | None": + if not isinstance(self.content, list): + return None + return next((item for item in self.content if item.tag == tag), None) + + def children(self, tag: str | None = None) -> list["BinaryNode"]: + if not isinstance(self.content, list): + return [] + return [item for item in self.content if tag is None or item.tag == tag] + + +class TokenTable: + """Versioned WABinary token lookup. + + Production can inject token tables generated from a pinned WhatsApp Web protocol snapshot. + Encoding always has a raw-string fallback, while decoding rejects unknown token references + instead of silently corrupting nodes. + """ + + def __init__(self, single: Sequence[str | None] = (), double: Sequence[Sequence[str | None]] = (), *, version: int | None = None): + self.single = tuple(single) + self.double = tuple(tuple(row) for row in double) + self.version = version + self._map: dict[str, tuple[int | None, int]] = {} + for idx, value in enumerate(self.single): + if value: + self._map[str(value)] = (None, idx) + for d, row in enumerate(self.double): + for idx, value in enumerate(row): + if value: + self._map[str(value)] = (d, idx) + + def token_for(self, value: str) -> tuple[int | None, int] | None: + return self._map.get(value) + + def decode_single(self, idx: int) -> str: + if not (0 <= idx < len(self.single)) or self.single[idx] is None: + raise BinaryNodeError(f"Unknown WABinary single-byte token {idx}") + return str(self.single[idx]) + + def decode_double(self, dictionary: int, idx: int) -> str: + if not (0 <= dictionary < len(self.double)) or not (0 <= idx < len(self.double[dictionary])) or self.double[dictionary][idx] is None: + raise BinaryNodeError(f"Unknown WABinary double-byte token {dictionary}:{idx}") + return str(self.double[dictionary][idx]) + + +EMPTY_TOKENS = TokenTable() + + +def _is_nibble(value: str) -> bool: + return bool(value) and len(value) <= TAGS.PACKED_MAX and all(ch.isdigit() or ch in "-." for ch in value) + + +def _is_hex(value: str) -> bool: + return bool(value) and len(value) <= TAGS.PACKED_MAX and all(ch in "0123456789ABCDEF" for ch in value) + + +def _pack(value: str, *, hex_mode: bool) -> bytes: + def cv(ch: str) -> int: + if ch == "\0": return 15 + if not hex_mode: + if ch == "-": return 10 + if ch == ".": return 11 + if ch.isdigit(): return ord(ch) - 48 + else: + if ch.isdigit(): return ord(ch) - 48 + if "A" <= ch <= "F": return 10 + ord(ch) - ord("A") + raise BinaryNodeError(f"Cannot pack character {ch!r}") + out = bytearray([TAGS.HEX_8 if hex_mode else TAGS.NIBBLE_8]) + size = (len(value) + 1) // 2 + out.append(size | (0x80 if len(value) % 2 else 0)) + for i in range(0, len(value), 2): + a = cv(value[i]) + b = cv(value[i + 1] if i + 1 < len(value) else "\0") + out.append((a << 4) | b) + return bytes(out) + + +def _unpack(tag: int, raw: bytes, pos: int) -> tuple[str, int]: + if pos >= len(raw): raise BinaryNodeError("Truncated packed string") + size_byte = raw[pos]; pos += 1 + nbytes = size_byte & 0x7F + odd = bool(size_byte & 0x80) + if pos + nbytes > len(raw): raise BinaryNodeError("Truncated packed string") + chars: list[str] = [] + for byte in raw[pos:pos+nbytes]: + for v in ((byte >> 4) & 15, byte & 15): + if tag == TAGS.HEX_8: + if v < 10: chars.append(chr(48 + v)) + elif v < 16: chars.append(chr(ord("A") + v - 10) if v < 15 else "\0") + else: raise BinaryNodeError("Invalid hex nibble") + else: + if v <= 9: chars.append(chr(48 + v)) + elif v == 10: chars.append("-") + elif v == 11: chars.append(".") + elif v == 15: chars.append("\0") + else: raise BinaryNodeError("Invalid numeric nibble") + if odd: + chars = chars[:-1] + return "".join(chars).replace("\0", ""), pos + nbytes + + +class BinaryNodeCodec: + def __init__(self, tokens: TokenTable = EMPTY_TOKENS, *, tags: BinaryTags = TAGS): + self.tokens = tokens + self.tags = tags + + def encode(self, node: BinaryNode, *, compressed: bool = False) -> bytes: + body = bytearray() + self._write_node(body, node) + if compressed: + return bytes([2]) + zlib.compress(bytes(body)) + return bytes([0]) + bytes(body) + + def decode(self, payload: bytes) -> BinaryNode: + raw = bytes(payload) + if not raw: + raise BinaryNodeError("Empty WABinary payload") + flag, body = raw[0], raw[1:] + if flag & 2: + try: + body = zlib.decompress(body) + except zlib.error as exc: + raise BinaryNodeError("Invalid compressed WABinary payload") from exc + node, pos = self._read_node(body, 0) + if pos != len(body): + raise BinaryNodeError(f"Trailing WABinary bytes: {len(body)-pos}") + return node + + def _list_start(self, out: bytearray, size: int) -> None: + if size == 0: out.append(self.tags.LIST_EMPTY) + elif size < 256: out.extend((self.tags.LIST_8, size)) + elif size < 65536: out.append(self.tags.LIST_16); out.extend(size.to_bytes(2, "big")) + else: raise BinaryNodeError("WABinary list too large") + + def _write_len(self, out: bytearray, length: int) -> None: + if length < 256: out.extend((self.tags.BINARY_8, length)) + elif length < (1 << 20): out.append(self.tags.BINARY_20); out.extend(((length >> 16) & 15, (length >> 8) & 255, length & 255)) + elif length < (1 << 32): out.append(self.tags.BINARY_32); out.extend(length.to_bytes(4, "big")) + else: raise BinaryNodeError("WABinary value too large") + + def _write_raw(self, out: bytearray, raw: bytes) -> None: + self._write_len(out, len(raw)); out.extend(raw) + + def _write_jid(self, out: bytearray, jid: JID) -> None: + if jid.device is not None: + out.extend((self.tags.AD_JID, jid.domain_type & 255, jid.device & 255)) + self._write_string(out, jid.user) + else: + out.append(self.tags.JID_PAIR) + if jid.user: self._write_string(out, jid.user) + else: out.append(self.tags.LIST_EMPTY) + self._write_string(out, jid.server) + + def _write_string(self, out: bytearray, value: str | None) -> None: + if value is None: + out.append(self.tags.LIST_EMPTY); return + token = self.tokens.token_for(value) + if token is not None: + dictionary, idx = token + if dictionary is not None: + if not 0 <= dictionary <= 3: raise BinaryNodeError("Token dictionary index out of range") + out.append(self.tags.DICTIONARY_0 + dictionary) + out.append(idx & 255); return + if _is_nibble(value): out.extend(_pack(value, hex_mode=False)); return + if _is_hex(value): out.extend(_pack(value, hex_mode=True)); return + try: + jid = parse_jid(value) + except ValueError: + jid = None + if jid is not None: + self._write_jid(out, jid); return + self._write_raw(out, value.encode("utf-8")) + + def _write_node(self, out: bytearray, node: BinaryNode) -> None: + if not node.tag: raise BinaryNodeError("Node tag cannot be empty") + attrs = [(str(k), str(v)) for k, v in (node.attrs or {}).items() if v is not None] + has_content = node.content is not None + self._list_start(out, 1 + 2 * len(attrs) + (1 if has_content else 0)) + self._write_string(out, node.tag) + for key, value in attrs: + self._write_string(out, key); self._write_string(out, value) + content = node.content + if content is None: return + if isinstance(content, str): self._write_string(out, content) + elif isinstance(content, (bytes, bytearray, memoryview)): self._write_raw(out, bytes(content)) + elif isinstance(content, list): + self._list_start(out, len(content)) + for child in content: + if not isinstance(child, BinaryNode): raise BinaryNodeError("Node child must be BinaryNode") + self._write_node(out, child) + else: raise BinaryNodeError(f"Unsupported node content: {type(content).__name__}") + + def _read_n(self, raw: bytes, pos: int, n: int) -> tuple[bytes, int]: + if pos + n > len(raw): raise BinaryNodeError("WABinary end of stream") + return raw[pos:pos+n], pos+n + + def _read_list_size(self, raw: bytes, pos: int, tag: int | None = None) -> tuple[int, int]: + if tag is None: + b, pos = self._read_n(raw, pos, 1); tag = b[0] + if tag == self.tags.LIST_EMPTY: return 0, pos + if tag == self.tags.LIST_8: + b, pos = self._read_n(raw, pos, 1); return b[0], pos + if tag == self.tags.LIST_16: + b, pos = self._read_n(raw, pos, 2); return int.from_bytes(b, "big"), pos + raise BinaryNodeError(f"Invalid WABinary list tag {tag}") + + def _read_len(self, raw: bytes, pos: int, tag: int) -> tuple[int, int]: + if tag == self.tags.BINARY_8: + b, pos = self._read_n(raw, pos, 1); return b[0], pos + if tag == self.tags.BINARY_20: + b, pos = self._read_n(raw, pos, 3); return ((b[0] & 15) << 16) | (b[1] << 8) | b[2], pos + if tag == self.tags.BINARY_32: + b, pos = self._read_n(raw, pos, 4); return int.from_bytes(b, "big"), pos + raise BinaryNodeError(f"Not a binary-length tag: {tag}") + + def _read_string(self, raw: bytes, pos: int, first: int | None = None) -> tuple[str, int]: + if first is None: + b, pos = self._read_n(raw, pos, 1); first = b[0] + if 1 <= first < len(self.tokens.single): return self.tokens.decode_single(first), pos + if self.tags.DICTIONARY_0 <= first <= self.tags.DICTIONARY_3: + b, pos = self._read_n(raw, pos, 1); return self.tokens.decode_double(first - self.tags.DICTIONARY_0, b[0]), pos + if first == self.tags.LIST_EMPTY: return "", pos + if first in (self.tags.BINARY_8, self.tags.BINARY_20, self.tags.BINARY_32): + length, pos = self._read_len(raw, pos, first); b, pos = self._read_n(raw, pos, length) + try: return b.decode("utf-8"), pos + except UnicodeDecodeError as exc: raise BinaryNodeError("Binary string is not UTF-8") from exc + if first in (self.tags.NIBBLE_8, self.tags.HEX_8): return _unpack(first, raw, pos) + if first == self.tags.JID_PAIR: + user, pos = self._read_string(raw, pos); server, pos = self._read_string(raw, pos) + if not server: raise BinaryNodeError("JID pair has empty server") + return f"{user}@{server}", pos + if first == self.tags.AD_JID: + b, pos = self._read_n(raw, pos, 2); domain_type, device = b[0], b[1] + user, pos = self._read_string(raw, pos) + server = {1:"lid",128:"hosted",129:"hosted.lid"}.get(domain_type, "s.whatsapp.net") + return str(JID(user, server, device=device, domain_type=domain_type)), pos + raise BinaryNodeError(f"Unknown WABinary string token/tag {first}") + + def _read_content(self, raw: bytes, pos: int, first: int) -> tuple[bytes | str | list[BinaryNode], int]: + if first in (self.tags.LIST_EMPTY, self.tags.LIST_8, self.tags.LIST_16): + size, pos = self._read_list_size(raw, pos, first) + items: list[BinaryNode] = [] + for _ in range(size): + node, pos = self._read_node(raw, pos); items.append(node) + return items, pos + if first in (self.tags.BINARY_8, self.tags.BINARY_20, self.tags.BINARY_32): + length, pos = self._read_len(raw, pos, first) + return self._read_n(raw, pos, length) + return self._read_string(raw, pos, first) + + def _read_node(self, raw: bytes, pos: int) -> tuple[BinaryNode, int]: + size, pos = self._read_list_size(raw, pos) + if size <= 0: raise BinaryNodeError("Invalid empty WABinary node") + tag, pos = self._read_string(raw, pos) + if not tag: raise BinaryNodeError("WABinary node tag is empty") + attrs: dict[str, str] = {} + for _ in range((size - 1) // 2): + key, pos = self._read_string(raw, pos); value, pos = self._read_string(raw, pos); attrs[key] = value + content = None + if size % 2 == 0: + b, pos = self._read_n(raw, pos, 1); content, pos = self._read_content(raw, pos, b[0]) + return BinaryNode(tag=tag, attrs=attrs, content=content), pos From a65b2f73ccff725d928e6f369654880b0ddd10f8 Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:20:57 +0200 Subject: [PATCH 013/139] Add v9.9.0 candidate runtime and WhatsApp service --- src/postmaster/calendar_v990.py | 421 ++++++++++++++++++++ src/postmaster/webgui_whatsapp_v990.py | 165 ++++++++ src/postmaster/whatsapp_runtime_v990.py | 121 ++++++ src/postmaster/whatsapp_v990/service.py | 183 +++++++++ src/postmaster/whatsapp_v990/signal_wire.py | 214 ++++++++++ 5 files changed, 1104 insertions(+) create mode 100644 src/postmaster/calendar_v990.py create mode 100644 src/postmaster/webgui_whatsapp_v990.py create mode 100644 src/postmaster/whatsapp_runtime_v990.py create mode 100644 src/postmaster/whatsapp_v990/service.py create mode 100644 src/postmaster/whatsapp_v990/signal_wire.py diff --git a/src/postmaster/calendar_v990.py b/src/postmaster/calendar_v990.py new file mode 100644 index 00000000..1bdf0279 --- /dev/null +++ b/src/postmaster/calendar_v990.py @@ -0,0 +1,421 @@ +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from email.message import Message +from email.policy import default +from email.parser import BytesParser +from typing import Any, Iterable +from urllib.parse import urlparse + +_CALENDAR_TYPES = { + "text/calendar", + "application/calendar", + "application/ics", + "application/icalendar", + "text/x-vcalendar", +} +_VIDEO_HOSTS = { + "meet.google.com", + "teams.microsoft.com", + "teams.live.com", + "zoom.us", + "www.zoom.us", + "webex.com", + "www.webex.com", + "whereby.com", + "jitsi.org", + "meet.jit.si", +} +_URL_RE = re.compile(r"https?://[^\s<>\"']+", re.I) + + +class CalendarError(ValueError): + pass + + +def _unfold_ical(text: str) -> list[str]: + normalized = text.replace("\r\n", "\n").replace("\r", "\n") + out: list[str] = [] + for line in normalized.split("\n"): + if line.startswith((" ", "\t")) and out: + out[-1] += line[1:] + else: + out.append(line) + return out + + +def _split_property(line: str) -> tuple[str, dict[str, str], str] | None: + if ":" not in line: + return None + left, value = line.split(":", 1) + bits = left.split(";") + name = bits[0].strip().upper() + if not name: + return None + params: dict[str, str] = {} + for bit in bits[1:]: + if "=" in bit: + k, v = bit.split("=", 1) + params[k.strip().upper()] = v.strip().strip('"') + return name, params, value.strip() + + +def _ical_unescape(value: str) -> str: + return ( + value.replace("\\N", "\n") + .replace("\\n", "\n") + .replace("\\,", ",") + .replace("\\;", ";") + .replace("\\\\", "\\") + ) + + +def _ical_escape(value: str) -> str: + return ( + str(value) + .replace("\\", "\\\\") + .replace(";", "\\;") + .replace(",", "\\,") + .replace("\r\n", "\\n") + .replace("\r", "\\n") + .replace("\n", "\\n") + ) + + +def _parse_dt(value: str, params: dict[str, str]) -> dict[str, Any]: + raw = value.strip() + timezone = params.get("TZID") or "" + date_only = params.get("VALUE", "").upper() == "DATE" or bool(re.fullmatch(r"\d{8}", raw)) + dt: datetime | None = None + try: + if date_only: + dt = datetime.strptime(raw[:8], "%Y%m%d").replace(tzinfo=UTC) + elif raw.endswith("Z"): + dt = datetime.strptime(raw, "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC) + elif re.fullmatch(r"\d{8}T\d{6}", raw): + # A TZID value is kept explicitly instead of pretending the local wall time is UTC. + dt = datetime.strptime(raw, "%Y%m%dT%H%M%S") + except ValueError: + dt = None + return { + "raw": raw, + "timezone": timezone or None, + "date_only": date_only, + "iso": dt.isoformat() if dt else None, + } + + +def _address(value: str) -> str: + v = value.strip() + if v.lower().startswith("mailto:"): + return v[7:] + return v + + +def _urls(values: Iterable[str]) -> list[str]: + out: list[str] = [] + seen: set[str] = set() + for value in values: + for match in _URL_RE.findall(value or ""): + url = match.rstrip(".,);]>") + if url not in seen: + seen.add(url) + out.append(url) + return out + + +def _video_urls(urls: Iterable[str]) -> list[str]: + result: list[str] = [] + for url in urls: + try: + host = (urlparse(url).hostname or "").lower() + except ValueError: + continue + if host in _VIDEO_HOSTS or any(host.endswith("." + item) for item in _VIDEO_HOSTS): + result.append(url) + continue + if any(token in host for token in ("zoom", "meet", "teams", "webex", "whereby", "jitsi")): + result.append(url) + return result + + +def parse_ics(text: str) -> dict[str, Any]: + """Parse VEVENT fields locally without resolving URLs or contacting organizers.""" + lines = _unfold_ical(text) + method = "" + events: list[dict[str, Any]] = [] + current: dict[str, Any] | None = None + raw_props: list[tuple[str, dict[str, str], str]] = [] + + for line in lines: + parsed = _split_property(line) + if not parsed: + continue + name, params, value = parsed + if name == "METHOD" and current is None: + method = value.upper() + continue + if name == "BEGIN" and value.upper() == "VEVENT": + current = { + "attendees": [], + "urls": [], + "video_urls": [], + "x_properties": {}, + } + raw_props = [] + continue + if name == "END" and value.upper() == "VEVENT" and current is not None: + text_values = [ + str(current.get("url") or ""), + str(current.get("location") or ""), + str(current.get("description") or ""), + ] + for key, val in current.get("x_properties", {}).items(): + if any(token in key for token in ("CONFERENCE", "MEETING", "HANGOUT", "ONLINE")): + text_values.append(str(val)) + urls = _urls(text_values) + current["urls"] = urls + current["video_urls"] = _video_urls(urls) + events.append(current) + current = None + raw_props = [] + continue + if current is None: + continue + raw_props.append(parsed) + clean = _ical_unescape(value) + if name == "UID": + current["uid"] = clean + elif name == "SUMMARY": + current["summary"] = clean + elif name == "DESCRIPTION": + current["description"] = clean + elif name == "LOCATION": + current["location"] = clean + elif name == "URL": + current["url"] = clean + elif name == "DTSTART": + current["start"] = _parse_dt(value, params) + elif name == "DTEND": + current["end"] = _parse_dt(value, params) + elif name == "DTSTAMP": + current["dtstamp"] = _parse_dt(value, params) + elif name == "SEQUENCE": + try: + current["sequence"] = int(value) + except ValueError: + current["sequence"] = value + elif name == "STATUS": + current["status"] = value.upper() + elif name == "ORGANIZER": + current["organizer"] = { + "address": _address(clean), + "name": params.get("CN") or None, + } + elif name == "ATTENDEE": + current["attendees"].append( + { + "address": _address(clean), + "name": params.get("CN") or None, + "role": params.get("ROLE") or None, + "partstat": params.get("PARTSTAT") or None, + "rsvp": params.get("RSVP") or None, + } + ) + elif name.startswith("X-"): + current["x_properties"][name] = clean + + invitation = bool(events) and method in {"REQUEST", "PUBLISH", "ADD", "CANCEL", "REPLY", "COUNTER", "DECLINECOUNTER"} + return { + "ok": True, + "method": method or None, + "is_calendar": bool(events), + "is_invitation": invitation, + "events": events, + "network_access": False, + "external_actions": False, + } + + +def _part_filename(part: Message) -> str: + try: + return str(part.get_filename() or "") + except Exception: + return "" + + +def detect_calendar_parts(message_or_bytes: Message | bytes) -> dict[str, Any]: + """Inspect headers and MIME parts. This function is deliberately zero-network/read-only.""" + message = ( + BytesParser(policy=default).parsebytes(message_or_bytes) + if isinstance(message_or_bytes, (bytes, bytearray)) + else message_or_bytes + ) + hints: list[str] = [] + headers = {str(k).lower(): str(v) for k, v in message.items()} + for name in ("content-class", "x-microsoft-cdo-message-class", "x-ms-has-attach"): + value = headers.get(name, "") + if "calend" in value.lower() or "appointment" in value.lower(): + hints.append(f"header:{name}") + + parts: list[dict[str, Any]] = [] + walk = list(message.walk()) if message.is_multipart() else [message] + for index, part in enumerate(walk): + content_type = str(part.get_content_type() or "").lower() + filename = _part_filename(part) + calendar_like = content_type in _CALENDAR_TYPES or filename.lower().endswith((".ics", ".ifb", ".vcs")) + if not calendar_like: + continue + try: + payload = part.get_payload(decode=True) + except Exception: + payload = None + if payload is None: + try: + content = part.get_content() + text = content if isinstance(content, str) else str(content) + except Exception: + text = "" + else: + charset = part.get_content_charset() or "utf-8" + try: + text = payload.decode(charset, errors="replace") + except LookupError: + text = payload.decode("utf-8", errors="replace") + parsed = parse_ics(text) if text else {"ok": True, "is_calendar": True, "is_invitation": False, "events": []} + parts.append( + { + "part_index": index, + "content_type": content_type, + "content_disposition": str(part.get_content_disposition() or ""), + "filename": filename or None, + "method_parameter": str(part.get_param("method") or "").upper() or None, + "sha256": hashlib.sha256((payload or text.encode("utf-8", "replace"))).hexdigest(), + "calendar": parsed, + } + ) + return { + "ok": True, + "calendar_candidate": bool(parts or hints), + "calendar_parts": parts, + "header_hints": hints, + "is_invitation": any(bool(p.get("calendar", {}).get("is_invitation")) for p in parts), + "network_access": False, + "external_actions": False, + } + + +def _fold_ical_line(line: str, limit: int = 75) -> list[str]: + # RFC 5545 fold limit is octets. Keep ASCII output where possible; UTF-8-safe fallback. + encoded = line.encode("utf-8") + if len(encoded) <= limit: + return [line] + out: list[str] = [] + current = "" + current_len = 0 + for ch in line: + size = len(ch.encode("utf-8")) + if current and current_len + size > (limit if not out else limit - 1): + out.append(current) + current = " " + ch + current_len = 1 + size + else: + current += ch + current_len += size + if current: + out.append(current) + return out + + +def build_calendar_request( + *, + uid: str, + summary: str, + start_utc: datetime, + end_utc: datetime, + organizer_email: str, + attendees: Iterable[str], + description: str = "", + location: str = "", + video_url: str | None = None, + sequence: int = 0, + prodid: str = "-//Postmaster MCP//v9.9//EN", +) -> str: + if start_utc.tzinfo is None or end_utc.tzinfo is None: + raise CalendarError("Calendar invitation start/end must be timezone-aware") + start = start_utc.astimezone(UTC) + end = end_utc.astimezone(UTC) + if end <= start: + raise CalendarError("Calendar invitation end must be after start") + attendee_list = [str(x).strip() for x in attendees if str(x).strip()] + if not attendee_list: + raise CalendarError("Calendar invitation requires at least one attendee") + if not organizer_email.strip(): + raise CalendarError("Calendar invitation requires organizer email") + + def stamp(dt: datetime) -> str: + return dt.astimezone(UTC).strftime("%Y%m%dT%H%M%SZ") + + desc = description + if video_url and video_url not in desc: + desc = (desc + "\n\n" if desc else "") + f"Video call: {video_url}" + lines = [ + "BEGIN:VCALENDAR", + "VERSION:2.0", + f"PRODID:{_ical_escape(prodid)}", + "CALSCALE:GREGORIAN", + "METHOD:REQUEST", + "BEGIN:VEVENT", + f"UID:{_ical_escape(uid)}", + f"DTSTAMP:{stamp(datetime.now(UTC))}", + f"DTSTART:{stamp(start)}", + f"DTEND:{stamp(end)}", + f"SEQUENCE:{int(sequence)}", + "STATUS:CONFIRMED", + f"SUMMARY:{_ical_escape(summary)}", + f"ORGANIZER:mailto:{organizer_email.strip()}", + ] + for attendee in attendee_list: + lines.append(f"ATTENDEE;ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:{attendee}") + if location: + lines.append(f"LOCATION:{_ical_escape(location)}") + if desc: + lines.append(f"DESCRIPTION:{_ical_escape(desc)}") + if video_url: + lines.append(f"URL:{video_url}") + lines.append(f"X-POSTMASTER-VIDEOCALL:{video_url}") + lines.extend(["END:VEVENT", "END:VCALENDAR"]) + folded: list[str] = [] + for line in lines: + folded.extend(_fold_ical_line(line)) + return "\r\n".join(folded) + "\r\n" + + +def job_calendar_window(job: dict[str, Any], *, default_duration_minutes: int = 30) -> tuple[datetime, datetime]: + """Resolve explicit calendar payload first, then one-time ISO schedule metadata.""" + payload = dict(job.get("payload") or {}) + cal = dict(payload.get("calendar") or {}) + start_raw = cal.get("start") or payload.get("calendar_start") + end_raw = cal.get("end") or payload.get("calendar_end") + if not start_raw and str(job.get("schedule_type") or "").lower() in {"once", "at", "datetime"}: + start_raw = job.get("schedule_value") + if not start_raw: + raise CalendarError("Task does not contain a concrete calendar start time") + + def parse(value: Any) -> datetime: + text = str(value or "").strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + dt = datetime.fromisoformat(text) + if dt.tzinfo is None: + raise CalendarError("Calendar task times must include a timezone offset") + return dt + + start = parse(start_raw) + end = parse(end_raw) if end_raw else start + timedelta(minutes=int(cal.get("duration_minutes") or default_duration_minutes)) + if end <= start: + raise CalendarError("Calendar task end must be after start") + return start, end diff --git a/src/postmaster/webgui_whatsapp_v990.py b/src/postmaster/webgui_whatsapp_v990.py new file mode 100644 index 00000000..a045a885 --- /dev/null +++ b/src/postmaster/webgui_whatsapp_v990.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import base64 +import html +from io import BytesIO +from typing import Any, Mapping +from urllib.parse import quote + +from starlette.requests import Request +from starlette.responses import PlainTextResponse, RedirectResponse +from starlette.routing import Mount, Route + + +VIEW = "whatsapp" +LABEL = "WhatsApp" + + +def _local_qr_data_uri(payload: str) -> str | None: + """Render QR locally when qrcode is available; never calls an external service.""" + try: + import qrcode + import qrcode.image.svg + qr = qrcode.QRCode(error_correction=qrcode.constants.ERROR_CORRECT_M, border=4, box_size=8) + qr.add_data(str(payload)); qr.make(fit=True) + image = qr.make_image(image_factory=qrcode.image.svg.SvgPathImage) + buf = BytesIO(); image.save(buf) + return "data:image/svg+xml;base64," + base64.b64encode(buf.getvalue()).decode("ascii") + except Exception: + return None + + +def qr_renderer_status() -> dict[str, object]: + try: + import qrcode # noqa: F401 + available = True + except Exception: + available = False + return {"local": True, "remote_service": False, "available": available} + + +def render_whatsapp_panel(status: Mapping[str, Any], *, qr_payload: str | None = None) -> str: + network = status.get("network") if isinstance(status.get("network"), Mapping) else {} + connected = bool(network.get("connected")) + paired = bool(status.get("paired")) + state = "Connected" if connected else ("Paired / disconnected" if paired else "Not paired") + qr_uri = _local_qr_data_uri(str(qr_payload)) if qr_payload else None + if qr_payload and qr_uri: + qr_block = ( + '' + ) + elif qr_payload: + qr_block = ( + '
Pairing QR renderer unavailable. ' + 'The transient pairing payload is retained encrypted server-side and is not exposed as plain text in this page.
' + ) + else: + qr_block = '

No active pairing QR. Start pairing explicitly to generate one.

' + return ( + '
' + '

WhatsApp

' + '

Clean-room companion session supervision.

v9.9.0
' + f'

Status: {html.escape(state)}

' + '

Reading messages here never sends read receipts. A read receipt is permitted only inside an explicit outbound reply flow.

' + f'{qr_block}' + '
' + ) + + +def _render_view(base: Any) -> str: + service = base.whatsapp_service_v990() + status = service.status() + qr = service.pairing_qr() + return '
' + render_whatsapp_panel(status, qr_payload=qr) + '
' + + +def _redirect(message: str = "") -> RedirectResponse: + suffix = ("?ui_view=whatsapp&flash=" + quote(message[:240])) if message else "?ui_view=whatsapp" + return RedirectResponse("/" + suffix + "#whatsapp", status_code=303) + + +async def _verified(base: Any, request: Request): + form, error = await base._verified_form(request) + if error: + return form, error + return form, None + + +async def start_pairing(base: Any, request: Request): + _form, error = await _verified(base, request) + if error: return error + try: + result = await base.whatsapp_service_v990().start_pairing() + message = "Pairing QR generated" if result.get("qr") else "Pairing started" + except Exception as exc: + message = f"{type(exc).__name__}: {exc}" + return _redirect(message) + + +async def reconnect(base: Any, request: Request): + _form, error = await _verified(base, request) + if error: return error + try: + result = await base.whatsapp_service_v990().reconnect() + message = "WhatsApp reconnected" if result.get("connected") or result.get("ok") else "Reconnect requested" + except Exception as exc: + message = f"{type(exc).__name__}: {exc}" + return _redirect(message) + + +def _insert_route(app: Any, route: Route) -> None: + for index, current in enumerate(app.router.routes): + if isinstance(current, Mount): + app.router.routes.insert(index, route); return + app.router.routes.append(route) + + +def _extend_nav(nav_module: Any) -> None: + groups = [] + found = False + for heading, links in nav_module.NAV_GROUPS: + mutable = list(links) + if any(item[0] == VIEW for item in mutable): found = True + if heading == "Communicate" and not found: + mutable.append((VIEW, LABEL, "WA")); found = True + groups.append((heading, tuple(mutable))) + if not found and groups: + heading, links = groups[-1]; groups[-1] = (heading, tuple(links) + ((VIEW, LABEL, "WA"),)) + nav_module.NAV_GROUPS = tuple(groups) + + +def install_webgui_whatsapp_v990(app: Any, base: Any, shell: Any, views: Any, nav_module: Any) -> None: + """Install the v9.9 WhatsApp supervision/pairing view into the existing lazy WebGUI shell.""" + old_render = shell.render_view + def render_with_whatsapp(bound_base: Any, core: Any, request: Request, view: str) -> str: + if view == VIEW: + return _render_view(bound_base) + return old_render(bound_base, core, request, view) + if VIEW not in shell.VIEWS: shell.VIEWS = tuple(shell.VIEWS) + (VIEW,) + if VIEW not in views.VIEWS: views.VIEWS = tuple(views.VIEWS) + (VIEW,) + shell.render_view = render_with_whatsapp + views.render_view = render_with_whatsapp + _extend_nav(nav_module) + + handlers = ( + ("/dashboard/whatsapp/pair", start_pairing, "v990_whatsapp_pair"), + ("/dashboard/whatsapp/reconnect", reconnect, "v990_whatsapp_reconnect"), + ) + existing = {getattr(route, "path", None) for route in app.router.routes} + for path, handler, name in handlers: + if path in existing: continue + async def endpoint(request: Request, _handler=handler): + if request.method != "POST": + return PlainTextResponse("Method Not Allowed", status_code=405, headers={"Allow": "POST"}) + return await _handler(base, request) + _insert_route(app, Route(path, endpoint, methods=["POST"], name=name)) + + +__all__ = [ + "VIEW", "LABEL", "qr_renderer_status", "render_whatsapp_panel", "start_pairing", "reconnect", "install_webgui_whatsapp_v990", +] diff --git a/src/postmaster/whatsapp_runtime_v990.py b/src/postmaster/whatsapp_runtime_v990.py new file mode 100644 index 00000000..321ced11 --- /dev/null +++ b/src/postmaster/whatsapp_runtime_v990.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +from typing import Any, Callable + +from mcp.types import ToolAnnotations + +from .whatsapp_v990.service import WhatsAppService, WhatsAppServiceError + +MCP_WHATSAPP_COMMANDS_V990 = 8 + + +def _ann(*, read_only: bool, idempotent: bool = False): + return ToolAnnotations(read_only_hint=read_only, destructive_hint=False, idempotent_hint=idempotent, open_world_hint=False) + + +def install_whatsapp_runtime_v990( + base: Any, + core: Any, + previous_runtime_status: Callable[[], Any], + *, + service: WhatsAppService | None = None, + auth_db: str = "/data/whatsapp-v990-auth.db", + event_db: str = "/data/whatsapp-v990-events.db", + key_path: str = "/data/whatsapp-v990-auth.key", +) -> dict[str, Any]: + """Install the v9.9 WhatsApp MCP boundary. + + The service defaults to a deliberately unavailable network adapter. Production release must + inject the verified current-protocol adapter after real QR/pair/send/receive interoperability + acceptance; tools fail closed until then. + """ + wa = service or WhatsAppService.create(auth_db=auth_db, event_db=event_db, key_path=key_path) + + def safe_sync(fn, *args, **kwargs): + try: return fn(*args, **kwargs) + except (WhatsAppServiceError, ValueError) as exc: return {"ok": False, "error": str(exc)} + except Exception as exc: return {"ok": False, "error": f"{type(exc).__name__}: {exc}"} + + async def safe_async(fn, *args, **kwargs): + try: return await fn(*args, **kwargs) + except (WhatsAppServiceError, ValueError) as exc: return {"ok": False, "error": str(exc)} + except Exception as exc: return {"ok": False, "error": f"{type(exc).__name__}: {exc}"} + + def whatsapp_status(): + """Read-only. Show WhatsApp pairing/network/session status without exposing private keys or secrets.""" + return safe_sync(wa.status) + + async def whatsapp_start_pairing(): + """WRITE ACTION. Explicitly start WhatsApp companion pairing and return the transient QR payload for WebGUI display.""" + return await safe_async(wa.start_pairing) + + async def whatsapp_reconnect(): + """WRITE ACTION. Explicitly reconnect the persisted WhatsApp companion session; never auto-pairs a new device.""" + return await safe_async(wa.reconnect) + + def whatsapp_list_messages(jid: str | None = None, limit: int = 100): + """Read-only. Read locally stored WhatsApp messages. Reading never emits a WhatsApp read receipt.""" + return safe_sync(wa.list_messages, jid=jid, limit=limit) + + async def whatsapp_send_text(jid: str, text: str, reply_to_message_id: str | None = None): + """WRITE ACTION. Send one WhatsApp text/group message. A read receipt may be emitted only when this explicit send is a reply.""" + return await safe_async(wa.send_text, jid=jid, text=text, reply_to_message_id=reply_to_message_id) + + async def whatsapp_send_media(jid: str, stored_file_id: str, caption: str = "", reply_to_message_id: str | None = None): + """WRITE ACTION. Send WhatsApp media from a Postmaster Stored File id; do not route raw attachment Base64 through the MCP call.""" + return await safe_async(wa.send_media, jid=jid, stored_file_id=stored_file_id, caption=caption, reply_to_message_id=reply_to_message_id) + + async def whatsapp_list_groups(): + """Read-only. List groups visible to the paired WhatsApp companion session.""" + return await safe_async(wa.list_groups) + + def whatsapp_list_receipts(limit: int = 200): + """Read-only. List remote receipts observed by Postmaster plus outbound-reply receipts emitted under the asymmetric policy.""" + return safe_sync(wa.list_receipts, limit=limit) + + tools = ( + ("whatsapp_status", whatsapp_status, _ann(read_only=True, idempotent=True)), + ("whatsapp_start_pairing", whatsapp_start_pairing, _ann(read_only=False)), + ("whatsapp_reconnect", whatsapp_reconnect, _ann(read_only=False)), + ("whatsapp_list_messages", whatsapp_list_messages, _ann(read_only=True, idempotent=True)), + ("whatsapp_send_text", whatsapp_send_text, _ann(read_only=False)), + ("whatsapp_send_media", whatsapp_send_media, _ann(read_only=False)), + ("whatsapp_list_groups", whatsapp_list_groups, _ann(read_only=True, idempotent=True)), + ("whatsapp_list_receipts", whatsapp_list_receipts, _ann(read_only=True, idempotent=True)), + ) + for name, fn, annotations in tools: + try: core.mcp.remove_tool(name) + except Exception: pass + core.mcp.add_tool(fn, name=name, annotations=annotations) + setattr(core, name, fn); setattr(base, name, fn) + + old_status = previous_runtime_status + def runtime_status(): + status = old_status() + status = dict(status) if isinstance(status, dict) else {"ok": True} + wa_status = wa.status() + status["whatsapp"] = { + "clean_room_python": True, + "tool_count": MCP_WHATSAPP_COMMANDS_V990, + "network_adapter_configured": bool(wa_status.get("network", {}).get("configured")), + "paired": bool(wa_status.get("paired")), + "read_receipts_asymmetric": True, + "local_read_emits_receipt": False, + "auth_encrypted_at_rest": True, + "private_material_exposed": False, + "protocol_interop_verified": False, + "signal_multidevice_verified": False, + "controlled_account_interop_verified": False, + } + return status + + # Runtime status is a replacement, not an additional command. + try: core.mcp.remove_tool("runtime_status") + except Exception: pass + core.mcp.add_tool(runtime_status, name="runtime_status", annotations=_ann(read_only=True, idempotent=True)) + setattr(core, "runtime_status", runtime_status); setattr(base, "runtime_status", runtime_status) + base.whatsapp_service_v990 = lambda: wa + return {"service": wa, "runtime_status": runtime_status, "tool_count": MCP_WHATSAPP_COMMANDS_V990} + + +__all__ = ["MCP_WHATSAPP_COMMANDS_V990", "install_whatsapp_runtime_v990"] diff --git a/src/postmaster/whatsapp_v990/service.py b/src/postmaster/whatsapp_v990/service.py new file mode 100644 index 00000000..b5954be0 --- /dev/null +++ b/src/postmaster/whatsapp_v990/service.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +from contextlib import closing +from dataclasses import dataclass +import json +from pathlib import Path +import sqlite3 +import threading +from typing import Any, Mapping, Protocol + +from .jid import parse_jid +from .receipts import receipt_for_event +from .store import EncryptedAuthStore + + +class WhatsAppServiceError(RuntimeError): + pass + + +class WhatsAppAdapter(Protocol): + async def start_pairing(self) -> Mapping[str, Any]: ... + async def reconnect(self) -> Mapping[str, Any]: ... + async def send_text(self, *, jid: str, text: str, reply_to_message_id: str | None, emit_read_receipt: bool) -> Mapping[str, Any]: ... + async def send_media(self, *, jid: str, stored_file_id: str, caption: str, reply_to_message_id: str | None, emit_read_receipt: bool) -> Mapping[str, Any]: ... + async def list_groups(self) -> list[Mapping[str, Any]]: ... + def status(self) -> Mapping[str, Any]: ... + + +class UnavailableAdapter: + """Safe default until the real current-protocol adapter is installed.""" + async def start_pairing(self): raise WhatsAppServiceError("WhatsApp network adapter is not configured") + async def reconnect(self): raise WhatsAppServiceError("WhatsApp network adapter is not configured") + async def send_text(self, **kwargs): raise WhatsAppServiceError("WhatsApp network adapter is not configured") + async def send_media(self, **kwargs): raise WhatsAppServiceError("WhatsApp network adapter is not configured") + async def list_groups(self): raise WhatsAppServiceError("WhatsApp network adapter is not configured") + def status(self): return {"configured": False, "connected": False} + + +class WhatsAppEventStore: + def __init__(self, path: str): + self.path = str(path) + Path(self.path).parent.mkdir(parents=True, exist_ok=True) + self._lock = threading.RLock() + self._init() + + def _connect(self): + conn = sqlite3.connect(self.path, timeout=30) + conn.row_factory = sqlite3.Row + return conn + + def _init(self): + with closing(self._connect()) as conn: + conn.executescript(""" + CREATE TABLE IF NOT EXISTS whatsapp_messages( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_id TEXT, + jid TEXT NOT NULL, + direction TEXT NOT NULL, + kind TEXT NOT NULL, + text TEXT NOT NULL DEFAULT '', + stored_file_id TEXT, + reply_to_message_id TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS idx_wa_messages_jid_id ON whatsapp_messages(jid,id DESC); + CREATE TABLE IF NOT EXISTS whatsapp_receipts( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_id TEXT, + jid TEXT, + receipt_type TEXT NOT NULL, + source TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + """) + conn.commit() + + def record_message(self, *, message_id: str | None, jid: str, direction: str, kind: str, text: str = "", stored_file_id: str | None = None, reply_to_message_id: str | None = None): + with self._lock, closing(self._connect()) as conn: + conn.execute("INSERT INTO whatsapp_messages(message_id,jid,direction,kind,text,stored_file_id,reply_to_message_id) VALUES(?,?,?,?,?,?,?)", + (message_id, jid, direction, kind, text, stored_file_id, reply_to_message_id)) + conn.commit() + + def list_messages(self, *, jid: str | None = None, limit: int = 100) -> list[dict[str, Any]]: + limit = max(1, min(int(limit), 500)) + with self._lock, closing(self._connect()) as conn: + if jid: + rows = conn.execute("SELECT * FROM whatsapp_messages WHERE jid=? ORDER BY id DESC LIMIT ?", (jid, limit)).fetchall() + else: + rows = conn.execute("SELECT * FROM whatsapp_messages ORDER BY id DESC LIMIT ?", (limit,)).fetchall() + return [dict(r) for r in rows] + + def record_receipt(self, *, message_id: str | None, jid: str | None, receipt_type: str, source: str): + with self._lock, closing(self._connect()) as conn: + conn.execute("INSERT INTO whatsapp_receipts(message_id,jid,receipt_type,source) VALUES(?,?,?,?)", (message_id,jid,receipt_type,source)) + conn.commit() + + def list_receipts(self, *, limit: int = 200) -> list[dict[str, Any]]: + limit = max(1, min(int(limit), 1000)) + with self._lock, closing(self._connect()) as conn: + rows = conn.execute("SELECT * FROM whatsapp_receipts ORDER BY id DESC LIMIT ?", (limit,)).fetchall() + return [dict(r) for r in rows] + + +@dataclass(slots=True) +class WhatsAppService: + auth: EncryptedAuthStore + events: WhatsAppEventStore + adapter: WhatsAppAdapter + + @classmethod + def create(cls, *, auth_db: str, event_db: str, key_path: str | None = None, adapter: WhatsAppAdapter | None = None) -> "WhatsAppService": + return cls(EncryptedAuthStore(auth_db, key_path=key_path), WhatsAppEventStore(event_db), adapter or UnavailableAdapter()) + + def status(self) -> dict[str, Any]: + network = dict(self.adapter.status()) + pairing = self.auth.get_json("runtime", "pairing") or {} + return { + "ok": True, + "network": network, + "paired": bool((self.auth.get_json("runtime", "identity") or {}).get("jid")), + "pairing_qr_available": bool(pairing.get("qr")), + "auth_store": self.auth.status(), + "read_receipts": {"incoming_visible": True, "local_read_emits": False, "outbound_reply_may_emit": True}, + } + + async def start_pairing(self) -> dict[str, Any]: + result = dict(await self.adapter.start_pairing()) + qr = result.get("qr") + if qr: + self.auth.put_json("runtime", "pairing", {"qr": str(qr), "expires_at": result.get("expires_at")}) + # Explicitly strip any private material the network adapter might accidentally return. + safe = {k:v for k,v in result.items() if k not in {"private_key","identity_private_key","noise_private_key","secret"}} + safe["private_material_exposed"] = False + return safe + + async def reconnect(self) -> dict[str, Any]: + return dict(await self.adapter.reconnect()) + + def pairing_qr(self) -> str | None: + state = self.auth.get_json("runtime", "pairing") or {} + return str(state.get("qr")) if state.get("qr") else None + + def list_messages(self, *, jid: str | None = None, limit: int = 100) -> dict[str, Any]: + if jid: jid = str(parse_jid(jid)) + return {"ok": True, "messages": self.events.list_messages(jid=jid, limit=limit), "read_receipt_emitted": False} + + async def send_text(self, *, jid: str, text: str, reply_to_message_id: str | None = None) -> dict[str, Any]: + dest = str(parse_jid(jid)) + if not str(text): raise WhatsAppServiceError("WhatsApp text cannot be empty") + decision = receipt_for_event(event="reply" if reply_to_message_id else "send", outbound_action=True, reply_to_message=bool(reply_to_message_id)) + result = dict(await self.adapter.send_text(jid=dest, text=str(text), reply_to_message_id=reply_to_message_id, emit_read_receipt=decision.emit)) + if result.get("ok", True): + self.events.record_message(message_id=result.get("message_id"), jid=dest, direction="out", kind="text", text=str(text), reply_to_message_id=reply_to_message_id) + if decision.emit: + self.events.record_receipt(message_id=reply_to_message_id, jid=dest, receipt_type="read", source="outbound_reply") + result["read_receipt_emitted"] = decision.emit + result["read_receipt_reason"] = decision.reason + return result + + async def send_media(self, *, jid: str, stored_file_id: str, caption: str = "", reply_to_message_id: str | None = None) -> dict[str, Any]: + dest = str(parse_jid(jid)) + fid = str(stored_file_id or "").strip() + if not fid: raise WhatsAppServiceError("stored_file_id is required; use Postmaster Stored File storage for WhatsApp media") + decision = receipt_for_event(event="reply" if reply_to_message_id else "send", outbound_action=True, reply_to_message=bool(reply_to_message_id)) + result = dict(await self.adapter.send_media(jid=dest, stored_file_id=fid, caption=str(caption or ""), reply_to_message_id=reply_to_message_id, emit_read_receipt=decision.emit)) + if result.get("ok", True): + self.events.record_message(message_id=result.get("message_id"), jid=dest, direction="out", kind="media", text=str(caption or ""), stored_file_id=fid, reply_to_message_id=reply_to_message_id) + if decision.emit: + self.events.record_receipt(message_id=reply_to_message_id, jid=dest, receipt_type="read", source="outbound_reply") + result["read_receipt_emitted"] = decision.emit + return result + + async def list_groups(self) -> dict[str, Any]: + return {"ok": True, "groups": [dict(x) for x in await self.adapter.list_groups()]} + + def record_incoming_message(self, *, message_id: str | None, jid: str, text: str = "", kind: str = "text", stored_file_id: str | None = None): + self.events.record_message(message_id=message_id, jid=str(parse_jid(jid)), direction="in", kind=kind, text=text, stored_file_id=stored_file_id) + + def record_incoming_receipt(self, *, message_id: str | None, jid: str | None, receipt_type: str): + self.events.record_receipt(message_id=message_id, jid=str(parse_jid(jid)) if jid else None, receipt_type=str(receipt_type), source="remote") + + def list_receipts(self, *, limit: int = 200) -> dict[str, Any]: + return {"ok": True, "receipts": self.events.list_receipts(limit=limit), "local_read_emits_receipt": False} diff --git a/src/postmaster/whatsapp_v990/signal_wire.py b/src/postmaster/whatsapp_v990/signal_wire.py new file mode 100644 index 00000000..95a06aa4 --- /dev/null +++ b/src/postmaster/whatsapp_v990/signal_wire.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +"""Clean-room Signal v3 wire messages used by the current WhatsApp libsignal path. + +This module deliberately implements only the stable, observable wire envelope used by +WhatsApp's current ``libsignal`` repository: Signal/Whisper messages and +PreKeyWhisper messages. It does not claim to be a full Signal session implementation. +""" + +from dataclasses import dataclass +import hashlib +import hmac + +from .crypto import signal_public_key +from .proto import ProtoError, decode_fields, field_bytes, field_varint + + +SIGNAL_V3 = 3 +SIGNAL_MAC_LENGTH = 8 + + +class SignalWireError(ValueError): + pass + + +def _version_byte(message_version: int = SIGNAL_V3, current_version: int = SIGNAL_V3) -> int: + if not 0 <= int(message_version) <= 15 or not 0 <= int(current_version) <= 15: + raise SignalWireError("Signal version nibbles must fit four bits") + return ((int(message_version) & 0xF) << 4) | (int(current_version) & 0xF) + + +def _parse_version(value: int, *, current_version: int = SIGNAL_V3) -> int: + high, low = (value >> 4) & 0xF, value & 0xF + if low != current_version: + raise SignalWireError(f"Unexpected Signal current-version nibble {low}") + if high < SIGNAL_V3: + raise SignalWireError(f"Legacy Signal message version {high}") + if high > current_version: + raise SignalWireError(f"Unsupported Signal message version {high}") + return high + + +def _field_map(data: bytes) -> dict[int, list[int | bytes]]: + try: + fields = decode_fields(data) + except ProtoError as exc: + raise SignalWireError("Invalid Signal protobuf encoding") from exc + out: dict[int, list[int | bytes]] = {} + for field in fields: + out.setdefault(field.number, []).append(field.value) + return out + + +def _required_bytes(fields: dict[int, list[int | bytes]], number: int, name: str) -> bytes: + values = fields.get(number) + if not values or not isinstance(values[-1], bytes): + raise SignalWireError(f"Missing Signal field {name}") + return bytes(values[-1]) + + +def _required_int(fields: dict[int, list[int | bytes]], number: int, name: str) -> int: + values = fields.get(number) + if not values or not isinstance(values[-1], int): + raise SignalWireError(f"Missing Signal field {name}") + value = int(values[-1]) + if not 0 <= value <= 0xFFFFFFFF: + raise SignalWireError(f"Signal field {name} is outside uint32 range") + return value + + +def _optional_int(fields: dict[int, list[int | bytes]], number: int) -> int | None: + values = fields.get(number) + if not values: + return None + if not isinstance(values[-1], int): + raise SignalWireError(f"Signal field {number} must be varint") + value = int(values[-1]) + if not 0 <= value <= 0xFFFFFFFF: + raise SignalWireError(f"Signal field {number} is outside uint32 range") + return value + + +def signal_message_mac( + *, + mac_key: bytes, + sender_identity_key: bytes, + receiver_identity_key: bytes, + serialized_without_mac: bytes, +) -> bytes: + if len(mac_key) != 32: + raise SignalWireError("Signal MAC key must be 32 bytes") + sender = signal_public_key(sender_identity_key) + receiver = signal_public_key(receiver_identity_key) + return hmac.new(bytes(mac_key), sender + receiver + bytes(serialized_without_mac), hashlib.sha256).digest()[:SIGNAL_MAC_LENGTH] + + +@dataclass(frozen=True, slots=True) +class SignalMessageV3: + ratchet_key: bytes + counter: int + previous_counter: int + ciphertext: bytes + mac: bytes = b"" + message_version: int = SIGNAL_V3 + + def protobuf(self) -> bytes: + ratchet = signal_public_key(self.ratchet_key) + return b"".join( + ( + field_bytes(1, ratchet), + field_varint(2, self.counter), + field_varint(3, self.previous_counter), + field_bytes(4, self.ciphertext), + ) + ) + + def unsigned_bytes(self) -> bytes: + return bytes((_version_byte(self.message_version),)) + self.protobuf() + + def serialize( + self, + *, + mac_key: bytes, + sender_identity_key: bytes, + receiver_identity_key: bytes, + ) -> bytes: + unsigned = self.unsigned_bytes() + mac = signal_message_mac( + mac_key=mac_key, + sender_identity_key=sender_identity_key, + receiver_identity_key=receiver_identity_key, + serialized_without_mac=unsigned, + ) + return unsigned + mac + + @classmethod + def parse(cls, raw: bytes) -> "SignalMessageV3": + data = bytes(raw) + if len(data) < 1 + SIGNAL_MAC_LENGTH: + raise SignalWireError("Signal message is too short") + version = _parse_version(data[0]) + proto = data[1:-SIGNAL_MAC_LENGTH] + mac = data[-SIGNAL_MAC_LENGTH:] + fields = _field_map(proto) + ratchet = _required_bytes(fields, 1, "ratchetKey") + if len(ratchet) != 33 or ratchet[0] != 5: + raise SignalWireError("Signal ratchet key must use 0x05-prefixed Curve25519 encoding") + return cls( + ratchet_key=ratchet, + counter=_required_int(fields, 2, "counter"), + previous_counter=_optional_int(fields, 3) or 0, + ciphertext=_required_bytes(fields, 4, "ciphertext"), + mac=mac, + message_version=version, + ) + + def verify_mac(self, *, mac_key: bytes, sender_identity_key: bytes, receiver_identity_key: bytes) -> bool: + expected = signal_message_mac( + mac_key=mac_key, + sender_identity_key=sender_identity_key, + receiver_identity_key=receiver_identity_key, + serialized_without_mac=self.unsigned_bytes(), + ) + return hmac.compare_digest(self.mac, expected) + + +@dataclass(frozen=True, slots=True) +class PreKeyWhisperMessageV3: + registration_id: int + pre_key_id: int | None + signed_pre_key_id: int + base_key: bytes + identity_key: bytes + message: bytes + message_version: int = SIGNAL_V3 + + def protobuf(self) -> bytes: + parts: list[bytes] = [] + if self.pre_key_id is not None: + parts.append(field_varint(1, self.pre_key_id)) + parts.extend( + ( + field_bytes(2, signal_public_key(self.base_key)), + field_bytes(3, signal_public_key(self.identity_key)), + field_bytes(4, self.message), + field_varint(5, self.registration_id), + field_varint(6, self.signed_pre_key_id), + ) + ) + return b"".join(parts) + + def serialize(self) -> bytes: + return bytes((_version_byte(self.message_version),)) + self.protobuf() + + @classmethod + def parse(cls, raw: bytes) -> "PreKeyWhisperMessageV3": + data = bytes(raw) + if len(data) < 2: + raise SignalWireError("PreKeyWhisper message is too short") + version = _parse_version(data[0]) + fields = _field_map(data[1:]) + base = _required_bytes(fields, 2, "baseKey") + ident = _required_bytes(fields, 3, "identityKey") + if len(base) != 33 or base[0] != 5 or len(ident) != 33 or ident[0] != 5: + raise SignalWireError("PreKeyWhisper keys must use 0x05-prefixed Curve25519 encoding") + return cls( + registration_id=_required_int(fields, 5, "registrationId"), + pre_key_id=_optional_int(fields, 1), + signed_pre_key_id=_required_int(fields, 6, "signedPreKeyId"), + base_key=base, + identity_key=ident, + message=_required_bytes(fields, 4, "message"), + message_version=version, + ) From 47580c256007b35bf81eee6003755a753e91643b Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:22:43 +0200 Subject: [PATCH 014/139] Add v9.9.0 tracking approval policy --- src/postmaster/tracking_policy_v990.py | 247 +++++++++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 src/postmaster/tracking_policy_v990.py diff --git a/src/postmaster/tracking_policy_v990.py b/src/postmaster/tracking_policy_v990.py new file mode 100644 index 00000000..2d365172 --- /dev/null +++ b/src/postmaster/tracking_policy_v990.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +from contextlib import closing + +import hashlib +import json +import secrets +import sqlite3 +import threading +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any, Mapping + +TRACKING_DISABLE_APPROVAL_TTL_SECONDS = 300 +GOOGLE_PROXY_HUMAN_DELAY_SECONDS = 60 + + +class TrackingApprovalError(ValueError): + pass + + +def _utcnow() -> datetime: + return datetime.now(UTC) + + +def _iso(dt: datetime) -> str: + return dt.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +def _parse_iso(value: str) -> datetime: + text = str(value or "").strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + return datetime.fromisoformat(text).astimezone(UTC) + + +def _normalize(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, Mapping): + return {str(k): _normalize(v) for k, v in sorted(value.items(), key=lambda item: str(item[0]))} + if isinstance(value, (list, tuple, set)): + return [_normalize(v) for v in value] + return str(value) + + +def canonical_intent_hash(intent: Mapping[str, Any]) -> str: + encoded = json.dumps(_normalize(dict(intent)), ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def body_digest(text: str | None) -> str: + return hashlib.sha256((text or "").encode("utf-8")).hexdigest() + + +def attachment_intent_digest(attachments: Any) -> str: + """Hash stable attachment descriptors; never persist attachment bytes in approval storage.""" + return canonical_intent_hash({"attachments": _normalize(attachments or [])}) + + +def effective_tracking(track_opens: bool | None, *, account_tracking_default: bool) -> bool: + return bool(account_tracking_default if track_opens is None else track_opens) + + +def generated_subject_guidance(subject: str, *, explicitly_user_supplied: bool = False) -> dict[str, Any]: + """Soft policy only: generated subjects prefer ASCII; user text is never silently rewritten.""" + non_ascii = [ch for ch in str(subject) if ord(ch) > 127] + return { + "subject": subject, + "ascii": not non_ascii, + "preferred_ascii": True, + "should_suggest_ascii": bool(non_ascii) and not explicitly_user_supplied, + "preserve_user_supplied_unicode": bool(explicitly_user_supplied), + } + + +class TrackingApprovalStore: + """Persistent one-use approvals bound to an exact outbound intent. + + The preview id is a correlation handle, not reusable consent. A successful consume + marks the record used before the outbound action is delegated. A failed send therefore + needs a fresh chat approval, matching the requested "every time" boundary. + """ + + def __init__(self, db_path: str | Path = "/data/tracking-approvals-v990.db", *, ttl_seconds: int = TRACKING_DISABLE_APPROVAL_TTL_SECONDS): + self.db_path = str(db_path) + self.ttl_seconds = int(ttl_seconds) + self._lock = threading.RLock() + Path(self.db_path).parent.mkdir(parents=True, exist_ok=True) + self._init_db() + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path, timeout=10.0, isolation_level=None) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA foreign_keys=ON") + return conn + + def _init_db(self) -> None: + with closing(self._connect()) as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS tracking_disable_approvals ( + id TEXT PRIMARY KEY, + intent_hash TEXT NOT NULL, + operation TEXT NOT NULL, + created_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + consumed_at TEXT, + intent_summary TEXT NOT NULL DEFAULT '{}' + ) + """ + ) + conn.execute("CREATE INDEX IF NOT EXISTS idx_tracking_disable_expiry ON tracking_disable_approvals(expires_at)") + + def create(self, *, operation: str, intent: Mapping[str, Any], summary: Mapping[str, Any] | None = None) -> dict[str, Any]: + now = _utcnow() + approval_id = "trkoff_" + secrets.token_urlsafe(18) + digest = canonical_intent_hash(intent) + expires = now + timedelta(seconds=self.ttl_seconds) + safe_summary = json.dumps(_normalize(dict(summary or {})), ensure_ascii=True, sort_keys=True, separators=(",", ":")) + with self._lock, closing(self._connect()) as conn: + conn.execute("DELETE FROM tracking_disable_approvals WHERE expires_at < ?", (_iso(now),)) + conn.execute( + "INSERT INTO tracking_disable_approvals(id,intent_hash,operation,created_at,expires_at,intent_summary) VALUES(?,?,?,?,?,?)", + (approval_id, digest, str(operation), _iso(now), _iso(expires), safe_summary), + ) + return { + "ok": True, + "approval_required": True, + "approval_type": "tracking_disabled", + "preview_id": approval_id, + "intent_hash": digest, + "expires_at": _iso(expires), + "one_use": True, + "send_performed": False, + "message": "Fresh explicit chat approval is required because the effective outbound tracking state is OFF.", + "summary": _normalize(dict(summary or {})), + } + + def consume(self, approval_id: str, *, operation: str, intent: Mapping[str, Any]) -> dict[str, Any]: + aid = str(approval_id or "").strip() + if not aid: + raise TrackingApprovalError("tracking_disable_approval_id is required after explicit chat approval") + expected = canonical_intent_hash(intent) + now = _utcnow() + with self._lock, closing(self._connect()) as conn: + conn.execute("BEGIN IMMEDIATE") + row = conn.execute("SELECT * FROM tracking_disable_approvals WHERE id=?", (aid,)).fetchone() + if row is None: + conn.execute("ROLLBACK") + raise TrackingApprovalError("Tracking-disable approval does not exist or has already expired") + if row["consumed_at"]: + conn.execute("ROLLBACK") + raise TrackingApprovalError("Tracking-disable approval has already been consumed") + if _parse_iso(row["expires_at"]) < now: + conn.execute("DELETE FROM tracking_disable_approvals WHERE id=?", (aid,)) + conn.execute("COMMIT") + raise TrackingApprovalError("Tracking-disable approval expired; ask for fresh explicit chat approval") + if str(row["operation"]) != str(operation) or str(row["intent_hash"]) != expected: + conn.execute("ROLLBACK") + raise TrackingApprovalError("Tracking-disable approval does not match this exact outbound action") + conn.execute("UPDATE tracking_disable_approvals SET consumed_at=? WHERE id=?", (_iso(now), aid)) + conn.execute("COMMIT") + return {"ok": True, "consumed": True, "preview_id": aid, "intent_hash": expected, "consumed_at": _iso(now)} + + +def classify_open_event( + *, + sent_at: str | datetime | None, + opened_at: str | datetime | None, + user_agent: str = "", + client_source: str = "", + threshold_seconds: int = GOOGLE_PROXY_HUMAN_DELAY_SECONDS, +) -> dict[str, Any]: + ua = str(user_agent or "").lower() + source = str(client_source or "").lower() + google_proxy = "googleimageproxy" in ua or "google_image_proxy" in source or "gmail_image_proxy" in source + + def parse_dt(value: str | datetime | None) -> datetime | None: + if value is None: + return None + if isinstance(value, datetime): + dt = value + else: + text = str(value).strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + dt = datetime.fromisoformat(text) + except ValueError: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt.astimezone(UTC) + + sent = parse_dt(sent_at) + opened = parse_dt(opened_at) + delay: float | None = None + if sent is not None and opened is not None: + delay = max(0.0, (opened - sent).total_seconds()) + + metadata_note = "Proxy-derived IP/location/browser/OS metadata is unreliable and must not be treated as end-user attribution." + if google_proxy: + if delay is not None and delay > float(threshold_seconds): + return { + "classification": "likely_human_via_proxy", + "confidence": "medium", + "google_image_proxy": True, + "delay_seconds": delay, + "threshold_seconds": int(threshold_seconds), + "reason": f"Google Image Proxy fetch occurred more than {int(threshold_seconds)} seconds after delivery; timing is consistent with a later human-triggered open, while the fetch remains proxied.", + "metadata_reliability": "low", + "metadata_note": metadata_note, + } + return { + "classification": "likely_machine_or_proxy", + "confidence": "medium" if delay is not None else "low", + "google_image_proxy": True, + "delay_seconds": delay, + "threshold_seconds": int(threshold_seconds), + "reason": f"Google Image Proxy fetch occurred within {int(threshold_seconds)} seconds of delivery or delivery timing is unavailable; prefetch/automation remains plausible.", + "metadata_reliability": "low", + "metadata_note": metadata_note, + } + if "proxy" in ua or "proxy" in source or "scanner" in ua or "scanner" in source: + return { + "classification": "uncertain_proxy_or_scanner", + "confidence": "low", + "google_image_proxy": False, + "delay_seconds": delay, + "threshold_seconds": int(threshold_seconds), + "reason": "Proxy/scanner indicators are present and do not establish a human read.", + "metadata_reliability": "low", + "metadata_note": metadata_note, + } + return { + "classification": "human_or_unclassified", + "confidence": "low", + "google_image_proxy": False, + "delay_seconds": delay, + "threshold_seconds": int(threshold_seconds), + "reason": "No known image-proxy/scanner signature was identified; an image fetch alone still does not prove a human read.", + "metadata_reliability": "medium", + "metadata_note": "Observed network/client metadata may still reflect relays, VPNs or security software.", + } From f904b49dc10c5b1bb06331e2d1071237b593f1ff Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:23:39 +0200 Subject: [PATCH 015/139] Add v9.9.0 hybrid email search index --- src/postmaster/email_search_v990.py | 445 ++++++++++++++++++++++++++++ 1 file changed, 445 insertions(+) create mode 100644 src/postmaster/email_search_v990.py diff --git a/src/postmaster/email_search_v990.py b/src/postmaster/email_search_v990.py new file mode 100644 index 00000000..98698d45 --- /dev/null +++ b/src/postmaster/email_search_v990.py @@ -0,0 +1,445 @@ +from __future__ import annotations + +from contextlib import closing + +import hashlib +import io +import json +import math +import re +import sqlite3 +import threading +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from email.message import Message +from email.parser import BytesParser +from email.policy import default +from html import unescape +from pathlib import Path +from typing import Any, Callable, Iterable, Mapping, Sequence + +import numpy as np + +try: + from pypdf import PdfReader +except Exception: # pragma: no cover - optional at import time in unit environments + PdfReader = None # type: ignore[assignment] + + +class EmailSearchError(ValueError): + pass + + +_TEXT_TYPES = { + "text/plain", + "text/csv", + "text/markdown", + "text/x-markdown", + "application/json", + "application/xml", + "text/xml", + "application/yaml", + "application/x-yaml", +} +_TOKEN_RE = re.compile(r"[\w@.+-]{2,}", re.UNICODE) +_TAG_RE = re.compile(r"<[^>]+>") + + +def _now_iso() -> str: + return datetime.now(UTC).isoformat().replace("+00:00", "Z") + + +def _clean_html(text: str) -> str: + text = re.sub(r"(?is)<(script|style)\b.*?>.*?", " ", text) + text = re.sub(r"(?i)|||", "\n", text) + text = _TAG_RE.sub(" ", text) + text = unescape(text) + return re.sub(r"[ \t]+", " ", re.sub(r"\n{3,}", "\n\n", text)).strip() + + +def _decode_part(part: Message, payload: bytes) -> str: + charset = part.get_content_charset() or "utf-8" + try: + return payload.decode(charset, errors="replace") + except LookupError: + return payload.decode("utf-8", errors="replace") + + +def extract_attachment_text(message_or_bytes: Message | bytes, *, max_total_chars: int = 200_000, max_part_chars: int = 80_000) -> list[dict[str, Any]]: + """Extract local text from supported MIME attachments; never resolve external URLs.""" + message = BytesParser(policy=default).parsebytes(message_or_bytes) if isinstance(message_or_bytes, (bytes, bytearray)) else message_or_bytes + results: list[dict[str, Any]] = [] + remaining = max(0, int(max_total_chars)) + for index, part in enumerate(message.walk() if message.is_multipart() else [message]): + filename = str(part.get_filename() or "") + disposition = str(part.get_content_disposition() or "") + if not filename and disposition != "attachment": + continue + ctype = str(part.get_content_type() or "").lower() + payload = part.get_payload(decode=True) or b"" + if not payload or remaining <= 0: + continue + text = "" + error = None + try: + if ctype in _TEXT_TYPES or filename.lower().endswith((".txt", ".md", ".csv", ".json", ".xml", ".yaml", ".yml")): + text = _decode_part(part, payload) + elif ctype == "text/html" or filename.lower().endswith((".html", ".htm")): + text = _clean_html(_decode_part(part, payload)) + elif ctype == "application/pdf" or filename.lower().endswith(".pdf"): + if PdfReader is None: + error = "pypdf unavailable" + else: + reader = PdfReader(io.BytesIO(payload)) + chunks: list[str] = [] + used = 0 + for page in reader.pages: + value = page.extract_text() or "" + chunks.append(value) + used += len(value) + if used >= max_part_chars: + break + text = "\n".join(chunks) + except Exception as exc: + error = f"{type(exc).__name__}: {exc}" + if text: + text = text[: min(max_part_chars, remaining)] + remaining -= len(text) + if text or error: + results.append( + { + "part_index": index, + "filename": filename or None, + "content_type": ctype, + "text": text, + "chars": len(text), + "error": error, + "sha256": hashlib.sha256(payload).hexdigest(), + "network_access": False, + } + ) + return results + + +def _vector_blob(vector: Sequence[float]) -> bytes: + arr = np.asarray(vector, dtype=np.float32).reshape(-1) + if arr.size == 0: + raise EmailSearchError("Semantic embedding is empty") + norm = float(np.linalg.norm(arr)) + if norm > 0: + arr = arr / norm + return arr.tobytes(order="C") + + +def _blob_vector(blob: bytes, dimensions: int) -> np.ndarray: + arr = np.frombuffer(blob, dtype=np.float32) + if arr.size != dimensions: + raise EmailSearchError("Stored email embedding dimension mismatch") + return arr + + +def _fts_query(query: str) -> str: + tokens = [t for t in _TOKEN_RE.findall(query or "") if t.strip(".-+")] + if not tokens: + return "" + # Quote every token to avoid exposing FTS syntax through MCP input. + return " AND ".join('"' + token.replace('"', '""') + '"' for token in tokens[:32]) + + +def _search_text(doc: Mapping[str, Any]) -> str: + attachments = doc.get("attachments_text") or "" + return "\n".join( + str(doc.get(key) or "") + for key in ("from_address", "to_address", "cc", "subject", "body_text", "snippet") + ) + ("\n" + str(attachments) if attachments else "") + + +class HybridEmailIndex: + """Persistent project-local email index: SQLite FTS5 + optional Model2Vec-compatible embeddings.""" + + def __init__( + self, + db_path: str | Path = "/data/email-search-v990.db", + *, + embed_one: Callable[[str], Sequence[float]] | None = None, + model_id: str = "postmaster-context-model", + ): + self.db_path = str(db_path) + self.embed_one = embed_one + self.model_id = str(model_id) + self._lock = threading.RLock() + Path(self.db_path).parent.mkdir(parents=True, exist_ok=True) + self._init_db() + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path, timeout=20.0) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA foreign_keys=ON") + return conn + + def _init_db(self) -> None: + with closing(self._connect()) as conn: + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS email_search_docs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + account_id TEXT NOT NULL, + mailbox TEXT NOT NULL, + uid TEXT NOT NULL, + message_id TEXT, + date_utc TEXT, + from_address TEXT, + to_address TEXT, + cc TEXT, + subject TEXT, + snippet TEXT, + body_text TEXT, + attachments_text TEXT, + attachments_json TEXT NOT NULL DEFAULT '[]', + content_hash TEXT NOT NULL, + indexed_at TEXT NOT NULL, + UNIQUE(account_id, mailbox, uid) + ); + CREATE VIRTUAL TABLE IF NOT EXISTS email_search_fts USING fts5( + from_address, to_address, cc, subject, snippet, body_text, attachments_text, + content='email_search_docs', content_rowid='id', tokenize='unicode61 remove_diacritics 2' + ); + CREATE TABLE IF NOT EXISTS email_search_embeddings ( + doc_id INTEGER PRIMARY KEY REFERENCES email_search_docs(id) ON DELETE CASCADE, + model_id TEXT NOT NULL, + dimensions INTEGER NOT NULL, + vector BLOB NOT NULL, + content_hash TEXT NOT NULL, + indexed_at TEXT NOT NULL + ); + CREATE TRIGGER IF NOT EXISTS email_search_docs_ai AFTER INSERT ON email_search_docs BEGIN + INSERT INTO email_search_fts(rowid,from_address,to_address,cc,subject,snippet,body_text,attachments_text) + VALUES(new.id,new.from_address,new.to_address,new.cc,new.subject,new.snippet,new.body_text,new.attachments_text); + END; + CREATE TRIGGER IF NOT EXISTS email_search_docs_ad AFTER DELETE ON email_search_docs BEGIN + INSERT INTO email_search_fts(email_search_fts,rowid,from_address,to_address,cc,subject,snippet,body_text,attachments_text) + VALUES('delete',old.id,old.from_address,old.to_address,old.cc,old.subject,old.snippet,old.body_text,old.attachments_text); + END; + CREATE TRIGGER IF NOT EXISTS email_search_docs_au AFTER UPDATE ON email_search_docs BEGIN + INSERT INTO email_search_fts(email_search_fts,rowid,from_address,to_address,cc,subject,snippet,body_text,attachments_text) + VALUES('delete',old.id,old.from_address,old.to_address,old.cc,old.subject,old.snippet,old.body_text,old.attachments_text); + INSERT INTO email_search_fts(rowid,from_address,to_address,cc,subject,snippet,body_text,attachments_text) + VALUES(new.id,new.from_address,new.to_address,new.cc,new.subject,new.snippet,new.body_text,new.attachments_text); + END; + CREATE INDEX IF NOT EXISTS idx_email_search_scope ON email_search_docs(account_id, mailbox, date_utc); + """ + ) + + def upsert(self, doc: Mapping[str, Any], *, embed: bool = True) -> dict[str, Any]: + account_id = str(doc.get("account_id") or "").strip() + mailbox = str(doc.get("mailbox") or "INBOX").strip() + uid = str(doc.get("uid") or "").strip() + if not account_id or not mailbox or not uid: + raise EmailSearchError("Email index requires account_id, mailbox and uid") + normalized = { + "message_id": str(doc.get("message_id") or ""), + "date_utc": str(doc.get("date_utc") or doc.get("date") or ""), + "from_address": str(doc.get("from_address") or doc.get("from") or ""), + "to_address": str(doc.get("to_address") or doc.get("to") or ""), + "cc": str(doc.get("cc") or ""), + "subject": str(doc.get("subject") or ""), + "snippet": str(doc.get("snippet") or ""), + "body_text": str(doc.get("body_text") or doc.get("body") or ""), + "attachments_text": str(doc.get("attachments_text") or ""), + } + attachment_meta = doc.get("attachments") or [] + content = _search_text(normalized) + content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest() + now = _now_iso() + with self._lock, closing(self._connect()) as conn: + existing = conn.execute( + "SELECT id,content_hash FROM email_search_docs WHERE account_id=? AND mailbox=? AND uid=?", + (account_id, mailbox, uid), + ).fetchone() + if existing and str(existing["content_hash"]) == content_hash: + doc_id = int(existing["id"]) + changed = False + else: + conn.execute( + """ + INSERT INTO email_search_docs( + account_id,mailbox,uid,message_id,date_utc,from_address,to_address,cc,subject,snippet,body_text, + attachments_text,attachments_json,content_hash,indexed_at + ) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + ON CONFLICT(account_id,mailbox,uid) DO UPDATE SET + message_id=excluded.message_id,date_utc=excluded.date_utc,from_address=excluded.from_address, + to_address=excluded.to_address,cc=excluded.cc,subject=excluded.subject,snippet=excluded.snippet, + body_text=excluded.body_text,attachments_text=excluded.attachments_text, + attachments_json=excluded.attachments_json,content_hash=excluded.content_hash,indexed_at=excluded.indexed_at + """, + ( + account_id, mailbox, uid, normalized["message_id"], normalized["date_utc"], normalized["from_address"], + normalized["to_address"], normalized["cc"], normalized["subject"], normalized["snippet"], + normalized["body_text"], normalized["attachments_text"], + json.dumps(attachment_meta, ensure_ascii=False, sort_keys=True), content_hash, now, + ), + ) + row = conn.execute( + "SELECT id FROM email_search_docs WHERE account_id=? AND mailbox=? AND uid=?", + (account_id, mailbox, uid), + ).fetchone() + doc_id = int(row["id"]) + conn.execute("DELETE FROM email_search_embeddings WHERE doc_id=?", (doc_id,)) + changed = True + semantic = False + if embed and self.embed_one is not None: + current = conn.execute( + "SELECT model_id,content_hash FROM email_search_embeddings WHERE doc_id=?", (doc_id,) + ).fetchone() + if not current or str(current["model_id"]) != self.model_id or str(current["content_hash"]) != content_hash: + vector = np.asarray(self.embed_one(content), dtype=np.float32).reshape(-1) + blob = _vector_blob(vector) + conn.execute( + "INSERT OR REPLACE INTO email_search_embeddings(doc_id,model_id,dimensions,vector,content_hash,indexed_at) VALUES(?,?,?,?,?,?)", + (doc_id, self.model_id, int(vector.size), blob, content_hash, now), + ) + semantic = True + conn.commit() + return {"ok": True, "doc_id": doc_id, "changed": changed, "semantic_indexed": semantic, "content_hash": content_hash} + + def search( + self, + query: str, + *, + account_id: str, + mailbox: str | None = None, + since_days: int | None = 90, + limit: int = 20, + semantic_weight: float = 0.60, + lexical_weight: float = 0.40, + semantic_scan_limit: int = 5000, + ) -> dict[str, Any]: + q = str(query or "").strip() + if not q: + raise EmailSearchError("Hybrid email search query is required") + limit = max(1, min(int(limit), 100)) + scope_sql = ["d.account_id=?"] + params: list[Any] = [str(account_id)] + if mailbox: + scope_sql.append("d.mailbox=?") + params.append(str(mailbox)) + if since_days is not None and int(since_days) >= 0: + cutoff = datetime.now(UTC) - timedelta(days=int(since_days)) + scope_sql.append("(d.date_utc='' OR d.date_utc IS NULL OR d.date_utc>=?)") + params.append(cutoff.isoformat()) + scope = " AND ".join(scope_sql) + lex: dict[int, tuple[int, float]] = {} + fts = _fts_query(q) + with closing(self._connect()) as conn: + if fts: + rows = conn.execute( + f""" + SELECT d.id, bm25(email_search_fts, 1.2,1.0,0.8,2.5,0.8,1.5,1.8) AS score + FROM email_search_fts JOIN email_search_docs d ON d.id=email_search_fts.rowid + WHERE email_search_fts MATCH ? AND {scope} + ORDER BY score ASC LIMIT ? + """, + [fts, *params, max(limit * 8, 100)], + ).fetchall() + for rank, row in enumerate(rows, 1): + lex[int(row["id"])] = (rank, float(row["score"])) + + sem: dict[int, tuple[int, float]] = {} + semantic_available = False + if self.embed_one is not None: + qv = np.asarray(self.embed_one(q), dtype=np.float32).reshape(-1) + qnorm = float(np.linalg.norm(qv)) + if qnorm > 0: + qv = qv / qnorm + rows = conn.execute( + f""" + SELECT d.id,e.dimensions,e.vector FROM email_search_docs d + JOIN email_search_embeddings e ON e.doc_id=d.id + WHERE e.model_id=? AND {scope} + ORDER BY d.date_utc DESC LIMIT ? + """, + [self.model_id, *params, max(1, min(int(semantic_scan_limit), 20000))], + ).fetchall() + scored: list[tuple[int, float]] = [] + for row in rows: + if int(row["dimensions"]) != int(qv.size): + continue + dv = _blob_vector(row["vector"], int(row["dimensions"])) + scored.append((int(row["id"]), float(np.dot(qv, dv)))) + scored.sort(key=lambda item: item[1], reverse=True) + sem = {doc_id: (rank, score) for rank, (doc_id, score) in enumerate(scored, 1)} + semantic_available = bool(scored) + + ids = set(lex) | set(sem) + fused: list[tuple[int, float]] = [] + k = 60.0 + for doc_id in ids: + score = 0.0 + if doc_id in lex: + score += float(lexical_weight) / (k + lex[doc_id][0]) + if doc_id in sem: + score += float(semantic_weight) / (k + sem[doc_id][0]) + fused.append((doc_id, score)) + fused.sort(key=lambda item: item[1], reverse=True) + selected = fused[:limit] + if not selected: + return { + "ok": True, "query": q, "results": [], "count": 0, + "semantic_active": semantic_available, "lexical_active": bool(fts), + "index_only": True, + } + placeholders = ",".join("?" for _ in selected) + docs = conn.execute( + f"SELECT * FROM email_search_docs WHERE id IN ({placeholders})", [doc_id for doc_id, _ in selected] + ).fetchall() + by_id = {int(row["id"]): row for row in docs} + results: list[dict[str, Any]] = [] + for doc_id, fused_score in selected: + row = by_id.get(doc_id) + if row is None: + continue + results.append( + { + "account_id": row["account_id"], "mailbox": row["mailbox"], "uid": row["uid"], + "message_id": row["message_id"], "date_utc": row["date_utc"], "from": row["from_address"], + "to": row["to_address"], "cc": row["cc"], "subject": row["subject"], "snippet": row["snippet"], + "body_indexed": bool(row["body_text"]), "attachment_text_indexed": bool(row["attachments_text"]), + "attachments": json.loads(row["attachments_json"] or "[]"), "score": fused_score, + "lexical_rank": lex.get(doc_id, (None, None))[0], "semantic_rank": sem.get(doc_id, (None, None))[0], + "semantic_similarity": sem.get(doc_id, (None, None))[1], + } + ) + return { + "ok": True, "query": q, "results": results, "count": len(results), + "semantic_active": semantic_available, "lexical_active": bool(fts), "index_only": True, + } + + def status(self) -> dict[str, Any]: + with closing(self._connect()) as conn: + docs = int(conn.execute("SELECT COUNT(*) FROM email_search_docs").fetchone()[0]) + vectors = int(conn.execute("SELECT COUNT(*) FROM email_search_embeddings WHERE model_id=?", (self.model_id,)).fetchone()[0]) + attachment_docs = int(conn.execute("SELECT COUNT(*) FROM email_search_docs WHERE attachments_text<>''").fetchone()[0]) + return { + "ok": True, "db_path": self.db_path, "documents": docs, "semantic_documents": vectors, + "attachment_text_documents": attachment_docs, "fts5": True, "model_id": self.model_id, + "external_resource_fetch": False, + } + + +def combine_attachment_text(extracted: Iterable[Mapping[str, Any]]) -> tuple[str, list[dict[str, Any]]]: + chunks: list[str] = [] + metadata: list[dict[str, Any]] = [] + for row in extracted: + text = str(row.get("text") or "") + filename = str(row.get("filename") or "") + if text: + chunks.append(f"[{filename or 'attachment'}]\n{text}") + metadata.append({ + "filename": filename or None, + "content_type": row.get("content_type"), + "chars": int(row.get("chars") or len(text)), + "sha256": row.get("sha256"), + "extraction_error": row.get("error"), + }) + return "\n\n".join(chunks), metadata From 917db5ae88bfc8c9a4278a7c31daf3e0685e1d97 Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:25:20 +0200 Subject: [PATCH 016/139] Add v9.9.0 email calendar runtime overlay --- src/postmaster/runtime_v990.py | 746 +++++++++++++++++++++++++++++++++ 1 file changed, 746 insertions(+) create mode 100644 src/postmaster/runtime_v990.py diff --git a/src/postmaster/runtime_v990.py b/src/postmaster/runtime_v990.py new file mode 100644 index 00000000..fbaa3a0c --- /dev/null +++ b/src/postmaster/runtime_v990.py @@ -0,0 +1,746 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +from datetime import UTC, datetime +from typing import Any, Callable, Mapping, Sequence + +from mcp.types import ToolAnnotations + +from .calendar_v990 import build_calendar_request, detect_calendar_parts, job_calendar_window +from .email_search_v990 import EmailSearchError, HybridEmailIndex, combine_attachment_text +from .tracking_policy_v990 import ( + TrackingApprovalError, + TrackingApprovalStore, + attachment_intent_digest, + body_digest, + classify_open_event, + effective_tracking, +) + +MCP_COMMAND_COUNT_V980 = 118 +MCP_EMAIL_CALENDAR_COMMANDS_V990 = 4 +# WhatsApp tools are installed by whatsapp_runtime_v990 and counted here for one release identity. +MCP_WHATSAPP_COMMANDS_V990 = 8 +MCP_COMMAND_COUNT_V990 = MCP_COMMAND_COUNT_V980 + MCP_EMAIL_CALENDAR_COMMANDS_V990 + MCP_WHATSAPP_COMMANDS_V990 + + +def _annotations(*, read_only: bool, destructive: bool = False, idempotent: bool = False): + return ToolAnnotations( + read_only_hint=read_only, + destructive_hint=destructive, + idempotent_hint=idempotent, + open_world_hint=False, + ) + + +def _as_dict(value: Any) -> dict[str, Any]: + return dict(value) if isinstance(value, Mapping) else {} + + +def _safe(fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: + try: + return fn(*args, **kwargs) + except (TrackingApprovalError, EmailSearchError, ValueError) as exc: + return {"ok": False, "error": str(exc)} + except Exception as exc: # preserve the project's MCP no-crash convention + return {"ok": False, "error": f"{type(exc).__name__}: {exc}"} + + +def _account(base: Any, account_id: str | None) -> dict[str, Any]: + store = base.account_store() + row = store.get_account(account_id) + if not isinstance(row, Mapping): + raise ValueError("Unable to resolve selected email account") + return dict(row) + + +def _account_id(base: Any, account_id: str | None) -> str: + row = _account(base, account_id) + resolved = str(row.get("id") or row.get("account_id") or "").strip() + if not resolved: + raise ValueError("Selected email account has no account id") + return resolved + + +def _tracking_default(base: Any, account_id: str | None) -> bool: + row = _account(base, account_id) + return bool(row.get("tracking_default", False)) + + +def _normalize_recipients(value: Sequence[str] | None) -> list[str]: + return [str(x).strip() for x in (value or []) if str(x).strip()] + + +def _send_intent( + operation: str, + *, + account_id: str, + to: Sequence[str] | None = None, + mailbox: str | None = None, + uid: str | None = None, + subject: str | None = None, + body: str = "", + body_html: str | None = None, + body_amp: str | None = None, + cc: Sequence[str] | None = None, + bcc: Sequence[str] | None = None, + attachments: Any = None, + campaign_id: str | None = None, + idempotency_key: str | None = None, + extras: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + return { + "operation": operation, + "account_id": account_id, + "to": _normalize_recipients(to), + "mailbox": str(mailbox or ""), + "uid": str(uid or ""), + "subject": str(subject or ""), + "body_sha256": body_digest(body), + "body_html_sha256": body_digest(body_html), + "body_amp_sha256": body_digest(body_amp), + "cc": _normalize_recipients(cc), + "bcc": _normalize_recipients(bcc), + "attachments_sha256": attachment_intent_digest(attachments), + "campaign_id": str(campaign_id or ""), + "idempotency_key": str(idempotency_key or ""), + "extras": dict(extras or {}), + } + + +def _approval_summary(intent: Mapping[str, Any]) -> dict[str, Any]: + return { + "operation": intent.get("operation"), + "account_id": intent.get("account_id"), + "to": intent.get("to"), + "mailbox": intent.get("mailbox"), + "uid": intent.get("uid"), + "subject": intent.get("subject"), + "tracking": "off", + } + + +def _guard_tracking_off( + store: TrackingApprovalStore, + *, + operation: str, + intent: Mapping[str, Any], + effective: bool, + approval_id: str | None, +) -> dict[str, Any] | None: + if effective: + return None + if not approval_id: + return store.create(operation=operation, intent=intent, summary=_approval_summary(intent)) + try: + store.consume(approval_id, operation=operation, intent=intent) + except TrackingApprovalError as exc: + return { + "ok": False, + "approval_required": True, + "approval_type": "tracking_disabled", + "send_performed": False, + "error": str(exc), + "message": "Ask for fresh explicit chat approval before retrying this tracking-OFF action.", + } + return None + + +def _semantic_embedder(base: Any) -> tuple[Callable[[str], Sequence[float]] | None, str]: + """Adapt the existing local semantic runtime without downloading a second model.""" + try: + engine = base.context_engine() + except Exception: + return None, "lexical-only" + candidates = [engine, getattr(engine, "semantic", None), getattr(engine, "semantic_engine", None), getattr(getattr(engine, "store", None), "semantic", None)] + for obj in [x for x in candidates if x is not None]: + model_id = str(getattr(obj, "model_id", None) or getattr(obj, "model_name", None) or "postmaster-context-model") + for name in ("embed_one", "encode_one"): + fn = getattr(obj, name, None) + if callable(fn): + return lambda text, fn=fn: fn(text), model_id + fn = getattr(obj, "encode", None) + if callable(fn): + def encode_one(text: str, fn=fn): + result = fn([text]) + try: + return result[0] + except Exception: + return result + return encode_one, model_id + model = getattr(obj, "model", None) + fn = getattr(model, "encode", None) + if callable(fn): + return lambda text, fn=fn: fn([text])[0], model_id + return None, "lexical-only" + + +def _message_rows(result: Any) -> list[dict[str, Any]]: + if isinstance(result, list): + return [dict(x) for x in result if isinstance(x, Mapping)] + if isinstance(result, Mapping): + for key in ("emails", "messages", "results", "items"): + value = result.get(key) + if isinstance(value, list): + return [dict(x) for x in value if isinstance(x, Mapping)] + return [] + + +def _full_message(result: Any) -> dict[str, Any]: + if isinstance(result, Mapping): + if isinstance(result.get("email"), Mapping): + return dict(result["email"]) + return dict(result) + return {} + + +def _attachment_text_from_tools(core: Any, *, mailbox: str, uid: str, account_id: str, max_attachments: int = 12) -> tuple[str, list[dict[str, Any]]]: + try: + listing = core.list_email_attachments(mailbox=mailbox, uid=uid, account_id=account_id) + except TypeError: + listing = core.list_email_attachments(mailbox, uid, account_id) + rows = [] + if isinstance(listing, Mapping): + rows = listing.get("attachments") or listing.get("items") or [] + elif isinstance(listing, list): + rows = listing + extracted: list[dict[str, Any]] = [] + for idx, item in enumerate(rows[:max_attachments]): + if not isinstance(item, Mapping): + continue + filename = str(item.get("filename") or item.get("name") or "") + content_type = str(item.get("content_type") or item.get("media_type") or "") + try: + read = core.read_email_attachment( + mailbox=mailbox, + uid=uid, + filename=filename or None, + index=item.get("index", idx), + max_chars=80000, + account_id=account_id, + ) + except Exception as exc: + extracted.append({"filename": filename or None, "content_type": content_type, "text": "", "error": f"{type(exc).__name__}: {exc}"}) + continue + if isinstance(read, Mapping): + text = str(read.get("text") or read.get("content") or "") + extracted.append({ + "filename": filename or read.get("filename"), + "content_type": content_type or read.get("content_type"), + "text": text, + "chars": len(text), + "error": read.get("error"), + "network_access": False, + }) + return combine_attachment_text(extracted) + + +def _enrich_email_index( + *, + base: Any, + core: Any, + index: HybridEmailIndex, + query: str, + account_id: str, + mailbox: str, + since_days: int, + enrich_limit: int, +) -> dict[str, Any]: + # Do not rely only on lexical IMAP TEXT matching: semantic queries may use different words. + try: + discovery = core.search_emails( + mailbox=mailbox, + since_days=since_days, + unread_only=False, + limit=max(1, min(int(enrich_limit), 100)), + account_id=account_id, + include_timings=False, + ) + except TypeError: + discovery = core.search_emails(mailbox=mailbox, since_days=since_days, limit=max(1, min(int(enrich_limit), 100)), account_id=account_id) + rows = _message_rows(discovery) + indexed = 0 + failures: list[dict[str, str]] = [] + for row in rows: + uid = str(row.get("uid") or "").strip() + if not uid: + continue + try: + detail = _full_message(core.get_email(mailbox=mailbox, uid=uid, account_id=account_id)) + attachments_text, attachment_meta = _attachment_text_from_tools(core, mailbox=mailbox, uid=uid, account_id=account_id) + doc = { + "account_id": account_id, + "mailbox": mailbox, + "uid": uid, + "message_id": detail.get("message_id") or row.get("message_id"), + "date_utc": detail.get("date_utc") or detail.get("date") or row.get("date_utc") or row.get("date"), + "from_address": detail.get("from_address") or detail.get("from") or row.get("from_address") or row.get("from"), + "to_address": detail.get("to_address") or detail.get("to") or row.get("to_address") or row.get("to"), + "cc": detail.get("cc") or row.get("cc"), + "subject": detail.get("subject") or row.get("subject"), + "snippet": detail.get("snippet") or row.get("snippet"), + "body_text": detail.get("body_text") or detail.get("body") or "", + "attachments_text": attachments_text, + "attachments": attachment_meta, + } + index.upsert(doc, embed=True) + indexed += 1 + except Exception as exc: + failures.append({"uid": uid, "error": f"{type(exc).__name__}: {exc}"}) + return {"indexed": indexed, "discovered": len(rows), "failures": failures[:20], "query": query, "network_scope": "imap_only"} + + +def _calendar_parts_from_detail(core: Any, *, mailbox: str, uid: str, account_id: str, get_email_fn: Callable[..., Any] | None = None) -> dict[str, Any]: + """Read calendar material locally from full-message/raw or ICS attachments. + + Existing Postmaster readers vary by historical release. Prefer raw MIME if exposed; otherwise + inspect calendar attachment bytes/text through the existing attachment tools. No remote URL is fetched. + """ + reader = get_email_fn or core.get_email + detail_raw = reader(mailbox=mailbox, uid=uid, account_id=account_id) + detail = _full_message(detail_raw) + raw = detail.get("raw_bytes") or detail.get("raw") or detail.get("raw_message") + if isinstance(raw, str): + try: + raw = base64.b64decode(raw, validate=True) + except Exception: + raw = raw.encode("utf-8", errors="replace") + if isinstance(raw, (bytes, bytearray)): + return detect_calendar_parts(bytes(raw)) + + found: list[dict[str, Any]] = [] + try: + listing = core.list_email_attachments(mailbox=mailbox, uid=uid, account_id=account_id) + except Exception: + listing = [] + rows = listing.get("attachments", []) if isinstance(listing, Mapping) else (listing if isinstance(listing, list) else []) + for idx, item in enumerate(rows): + if not isinstance(item, Mapping): + continue + filename = str(item.get("filename") or item.get("name") or "") + ctype = str(item.get("content_type") or item.get("media_type") or "").lower() + if ctype not in {"text/calendar", "application/ics", "application/calendar", "application/icalendar"} and not filename.lower().endswith((".ics", ".vcs", ".ifb")): + continue + try: + downloaded = core.get_email_attachment( + mailbox=mailbox, uid=uid, filename=filename or None, index=item.get("index", idx), include_base64=True, account_id=account_id + ) + if not isinstance(downloaded, Mapping): + continue + b64 = downloaded.get("content_base64") or downloaded.get("base64") or downloaded.get("data_base64") + if b64: + payload = base64.b64decode(str(b64)) + from email.message import EmailMessage + msg = EmailMessage() + msg["Subject"] = str(detail.get("subject") or "") + msg.set_content("calendar attachment") + msg.add_attachment(payload, maintype="text", subtype="calendar", filename=filename or "invite.ics") + parsed = detect_calendar_parts(msg.as_bytes()) + found.extend(parsed.get("calendar_parts", [])) + except Exception: + continue + return { + "ok": True, + "calendar_detected": bool(found), + "calendar_parts": found, + "network_access": False, + "external_actions": False, + "detection_scope": "raw-mime-or-local-calendar-attachments", + } + + +def install_runtime_v990( + base: Any, + core: Any, + previous_runtime_status: Callable[[], Any], + *, + tracking_approval_db: str | None = None, + email_search_db: str | None = None, +) -> dict[str, Any]: + """Install v9.9 email/calendar policy and search overlays; WhatsApp installs separately.""" + approval_store = TrackingApprovalStore(tracking_approval_db or "/data/tracking-approvals-v990.db") + embed_one, model_id = _semantic_embedder(base) + email_index = HybridEmailIndex(email_search_db or "/data/email-search-v990.db", embed_one=embed_one, model_id=model_id) + + old_send = core.send_email + old_reply = core.reply_email + old_follow = core.follow_up_email + old_list_open_events = core.list_open_events + old_get_email = core.get_email + old_list_tracking_deliveries = getattr(core, "list_tracking_deliveries", None) + + def send_email( + to: list[str], + subject: str, + body: str = "", + cc: list[str] | None = None, + bcc: list[str] | None = None, + body_html: str | None = None, + body_amp: str | None = None, + attachments: list[dict[str, Any]] | None = None, + track_opens: bool | None = None, + campaign_id: str | None = None, + account_id: str | None = None, + newsletter_mode: bool = False, + unsubscribe_url: str | None = None, + unsubscribe_email: str | None = None, + one_click_unsubscribe: bool = False, + automatic_unsubscribe: bool = True, + dsn_notify_success: bool = False, + idempotency_key: str | None = None, + force_send: bool = False, + confirm_suppressed_recipients: list[str] | None = None, + tracking_disable_approval_id: str | None = None, + ): + """WRITE ACTION. Send email through the existing outbound pipeline. + + Generated subjects should prefer ASCII text/punctuation when practical; never silently rewrite + Unicode explicitly supplied by the user. Prefer a Postmaster Stored File/signed storage link over + embedding an attachment unless the user requests an attachment or the recipient/workflow requires it. + + If the effective tracking state is OFF -- including account-default OFF when track_opens=None -- do + not send until the user gives fresh explicit chat approval for this exact send. First call returns a + one-use tracking_disable preview id; retry only after that approval with tracking_disable_approval_id. + Suppressed-recipient approval remains a separate exact per-send boundary. + """ + aid = _account_id(base, account_id) + eff = effective_tracking(track_opens, account_tracking_default=_tracking_default(base, account_id)) + extras = { + "newsletter_mode": newsletter_mode, + "unsubscribe_url": unsubscribe_url or "", + "unsubscribe_email": unsubscribe_email or "", + "one_click_unsubscribe": one_click_unsubscribe, + "automatic_unsubscribe": automatic_unsubscribe, + "dsn_notify_success": dsn_notify_success, + "force_send": force_send, + "confirm_suppressed_recipients": sorted(confirm_suppressed_recipients or []), + } + intent = _send_intent( + "send_email", account_id=aid, to=to, subject=subject, body=body, body_html=body_html, body_amp=body_amp, + cc=cc, bcc=bcc, attachments=attachments, campaign_id=campaign_id, idempotency_key=idempotency_key, extras=extras, + ) + blocked = _guard_tracking_off(approval_store, operation="send_email", intent=intent, effective=eff, approval_id=tracking_disable_approval_id) + if blocked: + return blocked + return old_send( + to=to, subject=subject, body=body, cc=cc, bcc=bcc, body_html=body_html, body_amp=body_amp, + attachments=attachments, track_opens=track_opens, campaign_id=campaign_id, account_id=account_id, + newsletter_mode=newsletter_mode, unsubscribe_url=unsubscribe_url, unsubscribe_email=unsubscribe_email, + one_click_unsubscribe=one_click_unsubscribe, automatic_unsubscribe=automatic_unsubscribe, + dsn_notify_success=dsn_notify_success, idempotency_key=idempotency_key, force_send=force_send, + confirm_suppressed_recipients=confirm_suppressed_recipients, + ) + + def _threaded(operation: str, delegate: Callable[..., Any], *, mailbox: str, uid: str, body: str, cc: list[str] | None, + bcc: list[str] | None, body_html: str | None, attachments: list[dict[str, Any]] | None, + track_opens: bool | None, campaign_id: str | None, account_id: str | None, newsletter_mode: bool, + unsubscribe_url: str | None, unsubscribe_email: str | None, one_click_unsubscribe: bool, + dsn_notify_success: bool, idempotency_key: str | None, force_send: bool, + confirm_suppressed_recipients: list[str] | None, tracking_disable_approval_id: str | None): + aid = _account_id(base, account_id) + eff = effective_tracking(track_opens, account_tracking_default=_tracking_default(base, account_id)) + extras = { + "newsletter_mode": newsletter_mode, + "unsubscribe_url": unsubscribe_url or "", + "unsubscribe_email": unsubscribe_email or "", + "one_click_unsubscribe": one_click_unsubscribe, + "dsn_notify_success": dsn_notify_success, + "force_send": force_send, + "confirm_suppressed_recipients": sorted(confirm_suppressed_recipients or []), + } + intent = _send_intent( + operation, account_id=aid, mailbox=mailbox, uid=uid, body=body, body_html=body_html, cc=cc, bcc=bcc, + attachments=attachments, campaign_id=campaign_id, idempotency_key=idempotency_key, extras=extras, + ) + blocked = _guard_tracking_off(approval_store, operation=operation, intent=intent, effective=eff, approval_id=tracking_disable_approval_id) + if blocked: + return blocked + return delegate( + mailbox=mailbox, uid=uid, body=body, cc=cc, bcc=bcc, body_html=body_html, attachments=attachments, + track_opens=track_opens, campaign_id=campaign_id, account_id=account_id, newsletter_mode=newsletter_mode, + unsubscribe_url=unsubscribe_url, unsubscribe_email=unsubscribe_email, one_click_unsubscribe=one_click_unsubscribe, + dsn_notify_success=dsn_notify_success, idempotency_key=idempotency_key, force_send=force_send, + confirm_suppressed_recipients=confirm_suppressed_recipients, + ) + + def reply_email( + mailbox: str, uid: str, body: str = "", cc: list[str] | None = None, bcc: list[str] | None = None, + body_html: str | None = None, attachments: list[dict[str, Any]] | None = None, track_opens: bool | None = None, + campaign_id: str | None = None, account_id: str | None = None, newsletter_mode: bool = False, + unsubscribe_url: str | None = None, unsubscribe_email: str | None = None, one_click_unsubscribe: bool = False, + dsn_notify_success: bool = False, idempotency_key: str | None = None, force_send: bool = False, + confirm_suppressed_recipients: list[str] | None = None, tracking_disable_approval_id: str | None = None, + ): + """WRITE ACTION. Reply in-thread through the existing outbound pipeline. + + Prefer Stored File/signed storage links over attachments unless an attachment is requested/required. + Any effective tracking-OFF reply requires fresh one-use explicit chat approval before transmission. + """ + return _threaded("reply_email", old_reply, mailbox=mailbox, uid=uid, body=body, cc=cc, bcc=bcc, + body_html=body_html, attachments=attachments, track_opens=track_opens, campaign_id=campaign_id, + account_id=account_id, newsletter_mode=newsletter_mode, unsubscribe_url=unsubscribe_url, + unsubscribe_email=unsubscribe_email, one_click_unsubscribe=one_click_unsubscribe, + dsn_notify_success=dsn_notify_success, idempotency_key=idempotency_key, force_send=force_send, + confirm_suppressed_recipients=confirm_suppressed_recipients, + tracking_disable_approval_id=tracking_disable_approval_id) + + def follow_up_email( + mailbox: str, uid: str, body: str = "", cc: list[str] | None = None, bcc: list[str] | None = None, + body_html: str | None = None, attachments: list[dict[str, Any]] | None = None, track_opens: bool | None = None, + campaign_id: str | None = None, account_id: str | None = None, newsletter_mode: bool = False, + unsubscribe_url: str | None = None, unsubscribe_email: str | None = None, one_click_unsubscribe: bool = False, + dsn_notify_success: bool = False, idempotency_key: str | None = None, force_send: bool = False, + confirm_suppressed_recipients: list[str] | None = None, tracking_disable_approval_id: str | None = None, + ): + """WRITE ACTION. Follow up on an outbound message through the existing pipeline. + + Prefer Stored File/signed storage links over attachments unless an attachment is requested/required. + Any effective tracking-OFF follow-up requires fresh one-use explicit chat approval before transmission. + """ + return _threaded("follow_up_email", old_follow, mailbox=mailbox, uid=uid, body=body, cc=cc, bcc=bcc, + body_html=body_html, attachments=attachments, track_opens=track_opens, campaign_id=campaign_id, + account_id=account_id, newsletter_mode=newsletter_mode, unsubscribe_url=unsubscribe_url, + unsubscribe_email=unsubscribe_email, one_click_unsubscribe=one_click_unsubscribe, + dsn_notify_success=dsn_notify_success, idempotency_key=idempotency_key, force_send=force_send, + confirm_suppressed_recipients=confirm_suppressed_recipients, + tracking_disable_approval_id=tracking_disable_approval_id) + + def _delivery_sent_times(*, delivery_id: str | None, campaign_id: str | None, recipient: str | None, account_id: str | None) -> dict[str, Any]: + if not callable(old_list_tracking_deliveries): + return {} + try: + deliveries = old_list_tracking_deliveries( + campaign_id=campaign_id, recipient=recipient, account_id=account_id, limit=1000 + ) + except TypeError: + try: + deliveries = old_list_tracking_deliveries(campaign_id=campaign_id, limit=1000) + except Exception: + return {} + except Exception: + return {} + candidates = deliveries if isinstance(deliveries, list) else (deliveries.get("deliveries", []) if isinstance(deliveries, Mapping) else []) + result: dict[str, Any] = {} + for item in candidates: + if not isinstance(item, Mapping): + continue + did = str(item.get("delivery_id") or item.get("id") or "").strip() + if delivery_id and did != str(delivery_id): + continue + if did: + result[did] = item.get("sent_at") or item.get("created_at") or item.get("delivered_at") + return result + + def list_open_events(delivery_id: str | None = None, campaign_id: str | None = None, recipient: str | None = None, + account_id: str | None = None, limit: int = 500): + """Read-only. List observed open-image events with conservative timing classification. + + Google Image Proxy events more than 60 seconds after sent_at are marked likely_human_via_proxy, not + guaranteed-human. If the open row lacks sent_at, Postmaster resolves it from the matching delivery. + Proxy-derived IP/location/browser/OS remain unreliable regardless of timing. + """ + result = old_list_open_events(delivery_id=delivery_id, campaign_id=campaign_id, recipient=recipient, account_id=account_id, limit=limit) + if not isinstance(result, (list, Mapping)): + return result + rows = result if isinstance(result, list) else result.get("events", result.get("open_events", [])) + if not isinstance(rows, list): + return result + sent_times = _delivery_sent_times(delivery_id=delivery_id, campaign_id=campaign_id, recipient=recipient, account_id=account_id) + for row in rows: + if not isinstance(row, dict): + continue + did = str(row.get("delivery_id") or "").strip() + sent_at = row.get("sent_at") or row.get("delivery_sent_at") or sent_times.get(did) + row["open_classification"] = classify_open_event( + sent_at=sent_at, opened_at=row.get("opened_at") or row.get("created_at"), + user_agent=str(row.get("user_agent") or ""), client_source=str(row.get("client_source") or ""), + ) + if isinstance(result, Mapping): + out = dict(result) + key = "events" if "events" in result else ("open_events" if "open_events" in result else None) + if key: + out[key] = rows + out["classification_policy"] = {"google_image_proxy_human_delay_seconds": 60, "human_is_probabilistic": True, "proxy_metadata_reliable": False} + return out + return rows + + def get_email(mailbox: str, uid: str, account_id: str | None = None, inspection: str | None = None, + content_mode: str = "safe", acknowledge_unsanitized_content_risk: bool = False): + """Read one email and always attach local MIME/header calendar-invitation analysis. + + Calendar detection is read-only and never accepts/declines an invite, creates a task, or fetches + remote content. Existing Safe Email content-mode semantics are preserved unchanged. + """ + kwargs = { + "mailbox": mailbox, "uid": uid, "account_id": account_id, "inspection": inspection, + "content_mode": content_mode, + "acknowledge_unsanitized_content_risk": acknowledge_unsanitized_content_risk, + } + try: + original = old_get_email(**kwargs) + except TypeError: + # Historical readers may not expose the later safe-inspection arguments. + original = old_get_email(mailbox=mailbox, uid=uid, account_id=account_id) + if not isinstance(original, Mapping): + return original + aid = _account_id(base, account_id) + analysis = _calendar_parts_from_detail( + core, mailbox=mailbox, uid=uid, account_id=aid, get_email_fn=old_get_email + ) + out = dict(original) + target = out.get("email") if isinstance(out.get("email"), Mapping) else out + if target is out: + out["calendar_analysis"] = analysis + else: + email_row = dict(target) + email_row["calendar_analysis"] = analysis + out["email"] = email_row + out["calendar_analysis_policy"] = { + "mandatory": True, "mime_and_headers": True, "network_access": False, + "automatic_rsvp": False, "automatic_task": False, "external_actions": False, + } + return out + + def email_search_status(account_id: str | None = None): + """Read-only. Return v9.9 email lexical/semantic index status for one account.""" + aid = _account_id(base, account_id) + result = email_index.status(account_id=aid) + result.update({"imap_on_demand_enrichment": True, "external_remote_resource_fetch": False, "attachment_text": True}) + return result + + def search_emails_hybrid(query: str, mailbox: str = "INBOX", since_days: int = 90, limit: int = 20, + account_id: str | None = None, enrich_imap: bool = True, enrich_limit: int = 50): + """Read-only. Hybrid FTS5 + local semantic email search over subject/body and extractable attachments. + + When enrich_imap=true, Postmaster may fetch a bounded set of missing recent messages/attachments from + the selected IMAP mailbox before ranking. It never loads external URLs/resources contained in email. + """ + aid = _account_id(base, account_id) + enrichment = {"indexed": 0, "discovered": 0, "failures": [], "network_scope": "none"} + if enrich_imap: + enrichment = _enrich_email_index(base=base, core=core, index=email_index, query=query, account_id=aid, + mailbox=mailbox, since_days=since_days, enrich_limit=enrich_limit) + result = email_index.search(query, account_id=aid, mailbox=mailbox, since_days=since_days, limit=limit) + result["enrichment"] = enrichment + result["external_remote_resource_fetch"] = False + return result + + def get_email_calendar_invites(mailbox: str, uid: str, account_id: str | None = None): + """Read-only. Analyze MIME/header/calendar material locally for invitations; never RSVP or communicate externally.""" + aid = _account_id(base, account_id) + result = _calendar_parts_from_detail(core, mailbox=mailbox, uid=uid, account_id=aid, get_email_fn=old_get_email) + result.update({"account_id": aid, "mailbox": mailbox, "uid": str(uid), "auto_rsvp": False, "auto_task": False}) + return result + + def send_job_calendar_invite(job_id: str, attendees: list[str] | None = None, account_id: str | None = None, + video_url: str | None = None, organizer_name: str | None = None, + track_opens: bool | None = None, tracking_disable_approval_id: str | None = None, + idempotency_key: str | None = None): + """WRITE ACTION. Explicitly send an RFC5545 calendar invitation for one stored passive task. + + create_job never sends. This tool is the explicit external action. A video URL already stored on the + task or supplied from another platform may be included. Tracking-OFF approval and recipient policy use + the normal email send boundary. + """ + job = core.get_job(job_id=job_id) + if not isinstance(job, Mapping): + return {"ok": False, "error": "Task not found"} + if isinstance(job.get("job"), Mapping): + job = job["job"] + payload = _as_dict(job.get("payload")) + cal = _as_dict(payload.get("calendar")) + recipients = _normalize_recipients(attendees or cal.get("attendees") or payload.get("attendees") or []) + if not recipients: + return {"ok": False, "error": "Calendar invitation requires at least one attendee"} + start_at, end_at = job_calendar_window(dict(job)) + title = str(cal.get("summary") or job.get("title") or "Scheduled task") + description = str(cal.get("description") or job.get("description") or "") + resolved_video = str(video_url or cal.get("video_url") or payload.get("video_url") or "").strip() or None + aid = _account_id(base, account_id) + account = _account(base, account_id) + organizer = str(account.get("email_address") or account.get("email") or "").strip() + if not organizer: + return {"ok": False, "error": "Selected account has no email address"} + uid = str(cal.get("uid") or f"postmaster-job-{job_id}@postmaster.local") + ics = build_calendar_request( + uid=uid, summary=title, start_utc=start_at, end_utc=end_at, organizer_email=organizer, + attendees=recipients, description=description, location=str(cal.get("location") or ""), + video_url=resolved_video, sequence=int(cal.get("sequence") or 0), + ) + attachment = { + "filename": "invite.ics", + "content_type": "text/calendar; method=REQUEST; charset=UTF-8", + "content_base64": base64.b64encode(ics.encode("utf-8")).decode("ascii"), + } + body = description or f"Calendar invitation: {title}" + if resolved_video: + body += f"\n\nVideo call: {resolved_video}" + return send_email( + to=recipients, subject=title, body=body, attachments=[attachment], track_opens=track_opens, + account_id=account_id, idempotency_key=idempotency_key or f"calendar-job:{job_id}:seq:{int(cal.get('sequence') or 0)}", + tracking_disable_approval_id=tracking_disable_approval_id, + ) + + def runtime_status(): + status = previous_runtime_status() + if not isinstance(status, dict): + status = {"ok": True} + status = dict(status) + status.update({ + "version_capability": "9.9.0", + "mcp_command_count_expected": MCP_COMMAND_COUNT_V990, + "mcp_command_count_delta_from_v980": MCP_EMAIL_CALENDAR_COMMANDS_V990 + MCP_WHATSAPP_COMMANDS_V990, + "tracking_disable_requires_fresh_chat_approval": True, + "tracking_disable_approval_one_use": True, + "google_image_proxy_likely_human_after_seconds": 60, + "proxy_attribution_metadata_reliable": False, + "subject_ascii_preferred": True, + "stored_file_link_preferred_over_attachment": True, + "calendar_invites": {"task_registry_passive": True, "explicit_send_only": True, "inbound_mime_local_analysis": True, "automatic_rsvp": False}, + "email_hybrid_search": {"fts5": True, "semantic": embed_one is not None, "attachments": True, "imap_on_demand": True, "external_resource_fetch": False}, + }) + return status + + # Replace existing names without changing their count; add four v9.9 email/calendar names. + replacements = ( + ("send_email", send_email, _annotations(read_only=False)), + ("reply_email", reply_email, _annotations(read_only=False)), + ("follow_up_email", follow_up_email, _annotations(read_only=False)), + ("list_open_events", list_open_events, _annotations(read_only=True, idempotent=True)), + ("get_email", get_email, _annotations(read_only=True, idempotent=True)), + ("runtime_status", runtime_status, _annotations(read_only=True, idempotent=True)), + ) + for name, fn, annotations in replacements: + core.mcp.remove_tool(name) + core.mcp.add_tool(fn, name=name, annotations=annotations) + setattr(core, name, fn) + setattr(base, name, fn) + additions = ( + ("email_search_status", email_search_status, _annotations(read_only=True, idempotent=True)), + ("search_emails_hybrid", search_emails_hybrid, _annotations(read_only=True, idempotent=True)), + ("get_email_calendar_invites", get_email_calendar_invites, _annotations(read_only=True, idempotent=True)), + ("send_job_calendar_invite", send_job_calendar_invite, _annotations(read_only=False)), + ) + for name, fn, annotations in additions: + core.mcp.add_tool(fn, name=name, annotations=annotations) + setattr(core, name, fn) + setattr(base, name, fn) + + base.tracking_approval_store_v990 = lambda: approval_store + base.email_search_index_v990 = lambda: email_index + return { + "approval_store": approval_store, + "email_index": email_index, + "runtime_status": runtime_status, + "mcp_command_count_expected": MCP_COMMAND_COUNT_V990, + } + + +__all__ = [ + "MCP_COMMAND_COUNT_V990", + "MCP_EMAIL_CALENDAR_COMMANDS_V990", + "MCP_WHATSAPP_COMMANDS_V990", + "install_runtime_v990", +] From b2b4019c4b09cfc7644e9f8debaab1fdd44b4c09 Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:25:42 +0200 Subject: [PATCH 017/139] Wire v9.9.0 candidate overlays into composed runtime --- src/postmaster/runtime.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/postmaster/runtime.py b/src/postmaster/runtime.py index 334d0f25..8872524c 100644 --- a/src/postmaster/runtime.py +++ b/src/postmaster/runtime.py @@ -25,6 +25,8 @@ from .runtime_v968 import install_runtime_v968 from .runtime_v969 import install_runtime_v969_mcp, install_runtime_v969_pre_webgui from .runtime_v980 import install_runtime_v980 +from .runtime_v990 import install_runtime_v990 +from .whatsapp_runtime_v990 import install_whatsapp_runtime_v990 from .tracking_telemetry_v971 import ( install_tracking_telemetry_v971, install_tracking_webgui_v971, @@ -39,6 +41,7 @@ ) from .webgui_release_identity import install_webgui_release_identity, project_release_version from .webgui_structured_data_v980 import install_webgui_structured_data_v980 +from .webgui_whatsapp_v990 import install_webgui_whatsapp_v990 from .webgui_tasks import task_fragment as _task_fragment_v945 from .webgui_v945 import install_webgui_v945 from .webgui_v951 import install_webgui_v951 @@ -54,6 +57,7 @@ from .webgui_v963_high_noise import install_webgui_v963_high_noise from .webgui_v964 import install_webgui_v964 from .webgui_v970 import install_webgui_v970 +from . import webgui_v970 as _webgui_v970 from .webgui_visual_restoration import install_webgui_visual_restoration for _name in dir(_core): @@ -161,6 +165,14 @@ def _task_dashboard_fragment(request): # unchanged. Destructive DDL is review-only and row deletes require explicit confirmation. runtime_status = install_runtime_v980(_base, _core, runtime_status) install_webgui_structured_data_v980(app, _base) +# v9.9.0 source overlays: email/calendar policy + hybrid search first, then the +# fail-closed clean-room WhatsApp surface. Real WhatsApp acceptance remains false +# until controlled protocol/multi-device interoperability has been demonstrated. +_v990 = install_runtime_v990(_base, _core, runtime_status) +runtime_status = _v990["runtime_status"] +_wa_v990 = install_whatsapp_runtime_v990(_base, _core, runtime_status) +runtime_status = _wa_v990["runtime_status"] +install_webgui_whatsapp_v990(app, _base, _webgui_v962, _webgui_v962_views, _webgui_v970) # The lazy shell originates in v9.6.2 but must identify the release that is actually loaded. # VERSION is local release metadata, so this does not add a network lookup to WebGUI rendering. install_webgui_release_identity(_webgui_v962, project_release_version()) From 6756343d4f0135f56bd233d2d144041c10dd0e21 Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:28:36 +0200 Subject: [PATCH 018/139] Restore exact r4 Signal/store sources --- src/postmaster/whatsapp_v990/signal_keys.py | 8 ++++---- src/postmaster/whatsapp_v990/store.py | Bin 5236 -> 5238 bytes 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/postmaster/whatsapp_v990/signal_keys.py b/src/postmaster/whatsapp_v990/signal_keys.py index 980213b2..90e1c44f 100644 --- a/src/postmaster/whatsapp_v990/signal_keys.py +++ b/src/postmaster/whatsapp_v990/signal_keys.py @@ -73,7 +73,7 @@ def derive_x3dh_initiator( ) -> tuple[bytes, bytes]: """Derive the legacy Signal-v3 initial root/chain pair for the initiator.""" pieces = [ - b"ÿ" * 32, + b"\xFF" * 32, curve_shared_key(our_identity_private, _raw_pub(their_signed_pre_key_public)), curve_shared_key(our_base_private, _raw_pub(their_identity_public)), curve_shared_key(our_base_private, _raw_pub(their_signed_pre_key_public)), @@ -94,7 +94,7 @@ def derive_x3dh_responder( ) -> tuple[bytes, bytes]: """Responder mirror of :func:`derive_x3dh_initiator`.""" pieces = [ - b"ÿ" * 32, + b"\xFF" * 32, curve_shared_key(our_signed_pre_key_private, _raw_pub(their_identity_public)), curve_shared_key(our_identity_private, _raw_pub(their_base_public)), curve_shared_key(our_signed_pre_key_private, _raw_pub(their_base_public)), @@ -108,13 +108,13 @@ def derive_x3dh_responder( def chain_message_seed(chain_key: bytes) -> bytes: if len(chain_key) != 32: raise SignalKeyError("Signal chain key must be 32 bytes") - return hmac.new(bytes(chain_key), b"", hashlib.sha256).digest() + return hmac.new(bytes(chain_key), b"\x01", hashlib.sha256).digest() def next_chain_key(chain_key: bytes) -> bytes: if len(chain_key) != 32: raise SignalKeyError("Signal chain key must be 32 bytes") - return hmac.new(bytes(chain_key), b"", hashlib.sha256).digest() + return hmac.new(bytes(chain_key), b"\x02", hashlib.sha256).digest() def derive_message_keys(chain_key: bytes) -> tuple[bytes, bytes, bytes]: diff --git a/src/postmaster/whatsapp_v990/store.py b/src/postmaster/whatsapp_v990/store.py index f42625f3259da0ec29d559713e9929761bffbef9..6da18711469e73aec62365f907c42738455bd1f9 100644 GIT binary patch delta 28 jcmeyO@l9icGZ%l1L3LhYZfbEsVsdJ2jKSsrE)^~Sq?rmo delta 26 hcmeyS@kL{UGZ!yIbzWj_YH>kga%wHZ<{&N=E&z(?2`2ym From 10312acee3b1d18372f2bccbfda147c8b6d778bc Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:28:51 +0200 Subject: [PATCH 019/139] Fix v9.9 email search status integration --- src/postmaster/runtime_v990.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/postmaster/runtime_v990.py b/src/postmaster/runtime_v990.py index fbaa3a0c..091086e6 100644 --- a/src/postmaster/runtime_v990.py +++ b/src/postmaster/runtime_v990.py @@ -606,8 +606,8 @@ def get_email(mailbox: str, uid: str, account_id: str | None = None, inspection: def email_search_status(account_id: str | None = None): """Read-only. Return v9.9 email lexical/semantic index status for one account.""" aid = _account_id(base, account_id) - result = email_index.status(account_id=aid) - result.update({"imap_on_demand_enrichment": True, "external_remote_resource_fetch": False, "attachment_text": True}) + result = email_index.status() + result.update({"account_id": aid, "imap_on_demand_enrichment": True, "external_remote_resource_fetch": False, "attachment_text": True}) return result def search_emails_hybrid(query: str, mailbox: str = "INBOX", since_days: int = 90, limit: int = 20, From 63bcfa7b13ed8c3d5e10966877d9aa698237f05e Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:29:21 +0200 Subject: [PATCH 020/139] Add v9.9.0 candidate regression tests --- tests/test_calendar_v990.py | 59 +++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tests/test_calendar_v990.py diff --git a/tests/test_calendar_v990.py b/tests/test_calendar_v990.py new file mode 100644 index 00000000..5a40c53f --- /dev/null +++ b/tests/test_calendar_v990.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from email.message import EmailMessage +import unittest + +from postmaster.calendar_v990 import build_calendar_request, detect_calendar_parts, parse_ics + + +class CalendarV990Tests(unittest.TestCase): + def test_roundtrip_request_and_video_link(self): + text = build_calendar_request( + uid="job-123@postmaster", + summary="Project sync", + start_utc=datetime(2026, 9, 21, 9, 0, tzinfo=UTC), + end_utc=datetime(2026, 9, 21, 9, 30, tzinfo=UTC), + organizer_email="owner@example.com", + attendees=["guest@example.com"], + description="Agenda", + video_url="https://meet.google.com/abc-defg-hij", + ) + parsed = parse_ics(text) + self.assertTrue(parsed["is_invitation"]) + event = parsed["events"][0] + self.assertEqual(event["uid"], "job-123@postmaster") + self.assertEqual(event["summary"], "Project sync") + self.assertEqual(event["attendees"][0]["address"], "guest@example.com") + self.assertEqual(event["video_urls"], ["https://meet.google.com/abc-defg-hij"]) + + def test_mime_inspection_is_local_and_detects_inline_calendar(self): + msg = EmailMessage() + msg["Subject"] = "Invite" + msg["From"] = "owner@example.com" + msg["To"] = "guest@example.com" + msg.set_content("Calendar invitation") + ics = build_calendar_request( + uid="u1@example.com", + summary="Demo", + start_utc=datetime(2026, 9, 21, 10, 0, tzinfo=UTC), + end_utc=datetime(2026, 9, 21, 11, 0, tzinfo=UTC), + organizer_email="owner@example.com", + attendees=["guest@example.com"], + ) + msg.add_alternative(ics, subtype="calendar", params={"method": "REQUEST"}) + result = detect_calendar_parts(msg.as_bytes()) + self.assertTrue(result["calendar_candidate"]) + self.assertTrue(result["is_invitation"]) + self.assertFalse(result["network_access"]) + self.assertFalse(result["external_actions"]) + + def test_cancel_is_invitation_semantics_without_external_action(self): + ics = "BEGIN:VCALENDAR\r\nMETHOD:CANCEL\r\nBEGIN:VEVENT\r\nUID:x\r\nSUMMARY:Cancelled\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n" + parsed = parse_ics(ics) + self.assertTrue(parsed["is_invitation"]) + self.assertFalse(parsed["external_actions"]) + + +if __name__ == "__main__": + unittest.main() From 7a61086785f6782bab278711c162737d52b16ba4 Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:29:24 +0200 Subject: [PATCH 021/139] Add v9.9.0 candidate regression tests --- tests/test_email_search_v990.py | 63 +++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 tests/test_email_search_v990.py diff --git a/tests/test_email_search_v990.py b/tests/test_email_search_v990.py new file mode 100644 index 00000000..55545852 --- /dev/null +++ b/tests/test_email_search_v990.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from email.message import EmailMessage +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest + +import numpy as np + +from postmaster.email_search_v990 import HybridEmailIndex, combine_attachment_text, extract_attachment_text + + +def fake_embed(text: str): + low = text.lower() + # Deliberately tiny deterministic semantic space for unit tests. + return np.array([ + low.count("invoice") + low.count("fattura"), + low.count("meeting") + low.count("riunione"), + low.count("printer") + low.count("stampante"), + ], dtype=np.float32) + + +class EmailSearchV990Tests(unittest.TestCase): + def test_hybrid_search_body_and_attachment_text(self): + with TemporaryDirectory() as td: + index = HybridEmailIndex(Path(td) / "index.db", embed_one=fake_embed, model_id="test") + index.upsert({ + "account_id": "a", "mailbox": "INBOX", "uid": "1", "subject": "Q3 document", + "body_text": "Here is the finance document", "attachments_text": "invoice total 1200 EUR", + "date_utc": "2026-09-20T10:00:00+00:00", + }) + index.upsert({ + "account_id": "a", "mailbox": "INBOX", "uid": "2", "subject": "Team meeting", + "body_text": "Weekly meeting", "date_utc": "2026-09-20T11:00:00+00:00", + }) + result = index.search("fattura", account_id="a", since_days=None) + self.assertEqual(result["results"][0]["uid"], "1") + self.assertTrue(result["semantic_active"]) + self.assertTrue(result["results"][0]["attachment_text_indexed"]) + + def test_local_attachment_extraction_never_fetches_network(self): + msg = EmailMessage() + msg["Subject"] = "Files" + msg.set_content("body") + msg.add_attachment(b"hello attachment", maintype="text", subtype="plain", filename="notes.txt") + extracted = extract_attachment_text(msg.as_bytes()) + self.assertEqual(extracted[0]["text"], "hello attachment") + self.assertFalse(extracted[0]["network_access"]) + combined, metadata = combine_attachment_text(extracted) + self.assertIn("hello attachment", combined) + self.assertEqual(metadata[0]["filename"], "notes.txt") + + def test_scope_is_account_bound(self): + with TemporaryDirectory() as td: + index = HybridEmailIndex(Path(td) / "index.db", embed_one=fake_embed, model_id="test") + index.upsert({"account_id": "a", "mailbox": "INBOX", "uid": "1", "subject": "invoice"}) + index.upsert({"account_id": "b", "mailbox": "INBOX", "uid": "1", "subject": "invoice secret"}) + result = index.search("invoice", account_id="a", since_days=None) + self.assertEqual({row["account_id"] for row in result["results"]}, {"a"}) + + +if __name__ == "__main__": + unittest.main() From 0e8ca7caa621570bc1cb2d67b8b0a70a557ffaad Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:29:26 +0200 Subject: [PATCH 022/139] Add v9.9.0 candidate regression tests --- tests/test_tracking_policy_v990.py | 66 ++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tests/test_tracking_policy_v990.py diff --git a/tests/test_tracking_policy_v990.py b/tests/test_tracking_policy_v990.py new file mode 100644 index 00000000..8998475d --- /dev/null +++ b/tests/test_tracking_policy_v990.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest + +from postmaster.tracking_policy_v990 import ( + TrackingApprovalError, + TrackingApprovalStore, + classify_open_event, + effective_tracking, + generated_subject_guidance, +) + + +class TrackingPolicyV990Tests(unittest.TestCase): + def test_effective_tracking_respects_account_default(self): + self.assertFalse(effective_tracking(None, account_tracking_default=False)) + self.assertTrue(effective_tracking(None, account_tracking_default=True)) + self.assertFalse(effective_tracking(False, account_tracking_default=True)) + self.assertTrue(effective_tracking(True, account_tracking_default=False)) + + def test_approval_is_exact_and_one_use(self): + with TemporaryDirectory() as td: + store = TrackingApprovalStore(Path(td) / "approvals.db") + intent = {"operation": "send_email", "to": ["a@example.com"], "subject": "Hello", "body_sha256": "abc"} + preview = store.create(operation="send_email", intent=intent, summary={"to": ["a@example.com"]}) + consumed = store.consume(preview["preview_id"], operation="send_email", intent=intent) + self.assertTrue(consumed["consumed"]) + with self.assertRaises(TrackingApprovalError): + store.consume(preview["preview_id"], operation="send_email", intent=intent) + + def test_approval_rejects_mutated_intent(self): + with TemporaryDirectory() as td: + store = TrackingApprovalStore(Path(td) / "approvals.db") + preview = store.create(operation="send_email", intent={"to": ["a@example.com"]}) + with self.assertRaises(TrackingApprovalError): + store.consume(preview["preview_id"], operation="send_email", intent={"to": ["b@example.com"]}) + + def test_google_proxy_after_one_minute_is_likely_human_but_metadata_low(self): + sent = datetime(2026, 9, 20, 10, 0, tzinfo=UTC) + result = classify_open_event( + sent_at=sent, + opened_at=sent + timedelta(seconds=61), + user_agent="Mozilla/5.0 GoogleImageProxy", + client_source="gmail_image_proxy", + ) + self.assertEqual(result["classification"], "likely_human_via_proxy") + self.assertEqual(result["metadata_reliability"], "low") + self.assertEqual(result["threshold_seconds"], 60) + + def test_google_proxy_at_one_minute_is_not_promoted(self): + sent = datetime(2026, 9, 20, 10, 0, tzinfo=UTC) + result = classify_open_event(sent_at=sent, opened_at=sent + timedelta(seconds=60), user_agent="GoogleImageProxy") + self.assertEqual(result["classification"], "likely_machine_or_proxy") + + def test_ascii_is_preference_not_rewrite(self): + result = generated_subject_guidance("Ciao — prova", explicitly_user_supplied=True) + self.assertEqual(result["subject"], "Ciao — prova") + self.assertFalse(result["should_suggest_ascii"]) + self.assertTrue(result["preserve_user_supplied_unicode"]) + + +if __name__ == "__main__": + unittest.main() From bef242670883d69395fd01ae22850c8914dc6f98 Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:29:56 +0200 Subject: [PATCH 023/139] Add v9.9.0 WhatsApp regression tests --- tests/test_whatsapp_crypto_v990.py | 62 ++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 tests/test_whatsapp_crypto_v990.py diff --git a/tests/test_whatsapp_crypto_v990.py b/tests/test_whatsapp_crypto_v990.py new file mode 100644 index 00000000..c280164b --- /dev/null +++ b/tests/test_whatsapp_crypto_v990.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import os +import unittest + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + +from postmaster.whatsapp_v990.crypto import ( + NOISE_WA_HEADER, + WhatsAppNoiseXX, + curve_shared_key, + decrypt_media, + encrypt_media, + frame_noise_payload, + generate_curve_keypair, + split_noise_frames, + xeddsa_public_edwards, + xeddsa_sign, + xeddsa_verify, +) + + +class WhatsAppCryptoV990Tests(unittest.TestCase): + def test_curve_shared_secret_matches(self): + a = generate_curve_keypair(); b = generate_curve_keypair() + self.assertEqual(curve_shared_key(a.private, b.public), curve_shared_key(b.private, a.public)) + + def test_xeddsa_signature_self_and_ed25519_verification(self): + pair = generate_curve_keypair() + msg = b"postmaster xeddsa compatibility" + sig = xeddsa_sign(pair.private, msg, random64=bytes(range(64))) + self.assertTrue(xeddsa_verify(pair.public, msg, sig)) + self.assertFalse(xeddsa_verify(pair.public, msg + b"!", sig)) + Ed25519PublicKey.from_public_bytes(xeddsa_public_edwards(pair.public)).verify(sig, msg) + + def test_media_roundtrip_and_tamper_rejection(self): + plaintext = os.urandom(913) + b"document" + enc = encrypt_media(plaintext, "document", media_key=bytes(range(32))) + self.assertEqual(decrypt_media(enc["encrypted"], "document", enc["media_key"], expected_file_sha256=enc["file_sha256"]), plaintext) + tampered = bytearray(enc["encrypted"]); tampered[-1] ^= 1 + with self.assertRaises(ValueError): + decrypt_media(bytes(tampered), "document", enc["media_key"]) + + def test_noise_frame_split_handles_partial_data(self): + payloads = [b"one", os.urandom(100), b""] + stream = b"".join(frame_noise_payload(p) for p in payloads) + frames, rest = split_noise_frames(stream[:-7]) + self.assertEqual(frames, payloads[:1]) + more, rest2 = split_noise_frames(rest + stream[-7:]) + self.assertEqual(more, payloads[1:]) + self.assertEqual(rest2, b"") + + def test_noise_symmetric_handshake_steps_are_deterministic(self): + # Full WA server certificate acceptance belongs to protocol tests. Here we validate + # that identical state transitions stay synchronized for encrypt/decrypt directions. + a = generate_curve_keypair() + n = WhatsAppNoiseXX(a, header=NOISE_WA_HEADER) + self.assertEqual(len(n.hash), 32) + self.assertEqual(n.counter, 0) + + +if __name__ == "__main__": unittest.main() From 68fbd21db9a86d8c4b4be0e121993c13cc320208 Mon Sep 17 00:00:00 2001 From: The Code Learner <142033899+the-code-learner@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:29:58 +0200 Subject: [PATCH 024/139] Add v9.9.0 WhatsApp regression tests --- tests/test_whatsapp_runtime_v990.py | 128 ++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 tests/test_whatsapp_runtime_v990.py diff --git a/tests/test_whatsapp_runtime_v990.py b/tests/test_whatsapp_runtime_v990.py new file mode 100644 index 00000000..a475a4c5 --- /dev/null +++ b/tests/test_whatsapp_runtime_v990.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +from pathlib import Path +import sys +import types +from tempfile import TemporaryDirectory +from types import SimpleNamespace +import unittest +from starlette.applications import Starlette + +mcp_mod = sys.modules.setdefault("mcp", types.ModuleType("mcp")) +mcp_types = sys.modules.setdefault("mcp.types", types.ModuleType("mcp.types")) +if not hasattr(mcp_types, "ToolAnnotations"): + class ToolAnnotations: + def __init__(self, **kwargs): self.__dict__.update(kwargs) + mcp_types.ToolAnnotations = ToolAnnotations + +from postmaster.whatsapp_runtime_v990 import MCP_WHATSAPP_COMMANDS_V990, install_whatsapp_runtime_v990 +from postmaster.whatsapp_v990.service import WhatsAppService, WhatsAppEventStore +from postmaster.whatsapp_v990.store import EncryptedAuthStore +from postmaster.webgui_whatsapp_v990 import render_whatsapp_panel, qr_renderer_status, install_webgui_whatsapp_v990 + + +class FakeMCP: + def __init__(self): self.tools={} + def remove_tool(self,name): self.tools.pop(name,None) + def add_tool(self,fn,name=None,annotations=None): self.tools[name or fn.__name__]=(fn,annotations) + + +class FakeAdapter: + def __init__(self): self.connected=False; self.calls=[] + async def start_pairing(self): self.calls.append("pair"); return {"ok":True,"qr":"qr-secret-payload","private_key":"never-return-this"} + async def reconnect(self): self.connected=True; self.calls.append("reconnect"); return {"ok":True,"connected":True} + async def send_text(self, **kwargs): self.calls.append(("text",kwargs)); return {"ok":True,"message_id":"m1"} + async def send_media(self, **kwargs): self.calls.append(("media",kwargs)); return {"ok":True,"message_id":"m2"} + async def list_groups(self): return [{"jid":"123@g.us","subject":"Test"}] + def status(self): return {"configured":True,"connected":self.connected} + + +class WhatsAppRuntimeTests(unittest.IsolatedAsyncioTestCase): + def make(self, td): + adapter=FakeAdapter() + service=WhatsAppService( + EncryptedAuthStore(str(Path(td)/"auth.db")), + WhatsAppEventStore(str(Path(td)/"events.db")), + adapter, + ) + core=SimpleNamespace(mcp=FakeMCP()) + base=SimpleNamespace() + out=install_whatsapp_runtime_v990(base,core,lambda:{"ok":True,"version_capability":"9.9.0","mcp_command_count_expected":130},service=service) + return base,core,adapter,service,out + + async def test_exact_eight_tools_and_safe_status(self): + with TemporaryDirectory() as td: + _,core,_,_,out=self.make(td) + expected={"whatsapp_status","whatsapp_start_pairing","whatsapp_reconnect","whatsapp_list_messages","whatsapp_send_text","whatsapp_send_media","whatsapp_list_groups","whatsapp_list_receipts"} + self.assertEqual(set(core.mcp.tools)-{"runtime_status"}, expected) + self.assertEqual(out["tool_count"], MCP_WHATSAPP_COMMANDS_V990) + status=core.whatsapp_status() + self.assertFalse(status["auth_store"]["private_material_exposed"]) + runtime=core.runtime_status() + self.assertTrue(runtime["whatsapp"]["read_receipts_asymmetric"]) + self.assertFalse(runtime["whatsapp"]["local_read_emits_receipt"]) + + async def test_pairing_strips_private_material_and_persists_qr(self): + with TemporaryDirectory() as td: + _,core,_,service,_=self.make(td) + result=await core.whatsapp_start_pairing() + self.assertNotIn("private_key", result) + self.assertFalse(result["private_material_exposed"]) + self.assertEqual(service.pairing_qr(), "qr-secret-payload") + + async def test_read_does_not_receipt_but_explicit_reply_may(self): + with TemporaryDirectory() as td: + _,core,adapter,service,_=self.make(td) + service.record_incoming_message(message_id="incoming1",jid="123@s.whatsapp.net",text="hello") + listed=core.whatsapp_list_messages() + self.assertFalse(listed["read_receipt_emitted"]) + self.assertEqual(core.whatsapp_list_receipts()["receipts"], []) + sent=await core.whatsapp_send_text("123@s.whatsapp.net","reply",reply_to_message_id="incoming1") + self.assertTrue(sent["read_receipt_emitted"]) + self.assertTrue(adapter.calls[-1][1]["emit_read_receipt"]) + receipts=core.whatsapp_list_receipts()["receipts"] + self.assertEqual(receipts[0]["source"],"outbound_reply") + + async def test_media_requires_stored_file_identifier(self): + with TemporaryDirectory() as td: + _,core,adapter,_,_=self.make(td) + bad=await core.whatsapp_send_media("123@s.whatsapp.net","") + self.assertFalse(bad["ok"]) + good=await core.whatsapp_send_media("123@g.us","file-123",caption="doc") + self.assertTrue(good["ok"]) + self.assertEqual(adapter.calls[-1][1]["stored_file_id"],"file-123") + + async def test_webgui_fragment_renders_qr_locally_without_remote_service(self): + panel=render_whatsapp_panel({"paired":False,"network":{"connected":False}},qr_payload='https://wa.me/settings/linked_devices#safe-test') + self.assertIn("WhatsApp pairing QR",panel) + self.assertNotIn("data-qr-payload",panel) + self.assertNotIn("wa.me/settings/linked_devices#safe-test",panel) + self.assertNotIn("