diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index c465a5da..4040c925 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -33,6 +33,23 @@ jobs: version = Path('VERSION').read_text(encoding='utf-8').strip() if not re.fullmatch(r'[0-9]+\.[0-9]+\.[0-9]+', version): raise SystemExit(f'Invalid VERSION: {version!r}') + if version == '9.9.0': + runtime = Path('src/postmaster/whatsapp_runtime_v990.py').read_text(encoding='utf-8') + required_acceptance = ( + '"protocol_interop_verified": True', + '"signal_multidevice_verified": True', + '"controlled_account_interop_verified": True', + ) + missing_acceptance = [ + item for item in required_acceptance if item not in runtime + ] + if missing_acceptance: + raise SystemExit( + 'Refusing stable v9.9.0 release without reviewed controlled-account ' + f'WhatsApp acceptance flags: {missing_acceptance}' + ) + print('WhatsApp v9.9.0 controlled-account acceptance flags verified.') + changelog = Path('CHANGELOG.md').read_text(encoding='utf-8') match = re.search( rf'^## {re.escape(version)} - .*?\n(?P.*?)(?=^## |\Z)', diff --git a/.github/workflows/whatsapp-v990-controlled-interop.yml b/.github/workflows/whatsapp-v990-controlled-interop.yml new file mode 100644 index 00000000..e2f96850 --- /dev/null +++ b/.github/workflows/whatsapp-v990-controlled-interop.yml @@ -0,0 +1,113 @@ +name: WhatsApp v9.9 controlled account interop + +on: + workflow_dispatch: + inputs: + require_inbound_echo: + description: Require controlled direct peer to echo the direct probe token + required: true + default: true + type: boolean + require_group_echo: + description: Require controlled peer to echo the group probe token + required: true + default: true + type: boolean + require_inbound_media: + description: Require controlled peer media in the test group and Stored File handoff + required: true + default: true + type: boolean + require_remote_receipt: + description: Require a remote delivery/read receipt for the direct probe text + required: true + default: true + type: boolean + run_media: + description: Exercise direct/group encrypted media upload and send + required: true + default: true + type: boolean + observe_timeout_seconds: + description: Seconds per controlled peer observation window (10-300) + required: true + default: '120' + type: string + +permissions: + contents: read + +concurrency: + group: whatsapp-v990-controlled-interop + cancel-in-progress: false + +jobs: + controlled-account: + name: WhatsApp controlled-account full interoperability + runs-on: ubuntu-latest + timeout-minutes: 14 + environment: whatsapp-v990-controlled + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install repository runtime dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + + - name: Validate controlled-session secrets are configured + env: + AUTH_DB: ${{ secrets.POSTMASTER_WA_AUTH_DB_B64 }} + AUTH_KEY: ${{ secrets.POSTMASTER_WA_AUTH_KEY_B64 }} + TEST_JID: ${{ secrets.POSTMASTER_WA_TEST_JID }} + TEST_GROUP_JID: ${{ secrets.POSTMASTER_WA_TEST_GROUP_JID }} + run: | + test -n "$AUTH_DB" || { echo "POSTMASTER_WA_AUTH_DB_B64 is not configured in the whatsapp-v990-controlled environment"; exit 2; } + test -n "$AUTH_KEY" || { echo "POSTMASTER_WA_AUTH_KEY_B64 is not configured in the whatsapp-v990-controlled environment"; exit 2; } + test -n "$TEST_JID" || { echo "POSTMASTER_WA_TEST_JID is not configured in the whatsapp-v990-controlled environment"; exit 2; } + test -n "$TEST_GROUP_JID" || { echo "POSTMASTER_WA_TEST_GROUP_JID is not configured in the whatsapp-v990-controlled environment"; exit 2; } + echo "Controlled-session secrets are configured (values not printed)." + + - name: Compile controlled interop harness + run: | + python -m compileall -q src/postmaster/whatsapp_v990 scripts/whatsapp_v990_controlled_interop.py + + - name: Run controlled direct/group text/media/receipt/inbound interoperability + env: + PYTHONPATH: src + POSTMASTER_WA_AUTH_DB_B64: ${{ secrets.POSTMASTER_WA_AUTH_DB_B64 }} + POSTMASTER_WA_AUTH_KEY_B64: ${{ secrets.POSTMASTER_WA_AUTH_KEY_B64 }} + POSTMASTER_WA_TEST_JID: ${{ secrets.POSTMASTER_WA_TEST_JID }} + POSTMASTER_WA_TEST_GROUP_JID: ${{ secrets.POSTMASTER_WA_TEST_GROUP_JID }} + POSTMASTER_WA_REQUIRE_INBOUND_ECHO: ${{ inputs.require_inbound_echo }} + POSTMASTER_WA_REQUIRE_GROUP_ECHO: ${{ inputs.require_group_echo }} + POSTMASTER_WA_REQUIRE_INBOUND_MEDIA: ${{ inputs.require_inbound_media }} + POSTMASTER_WA_REQUIRE_REMOTE_RECEIPT: ${{ inputs.require_remote_receipt }} + POSTMASTER_WA_RUN_MEDIA: ${{ inputs.run_media }} + POSTMASTER_WA_OBSERVE_TIMEOUT_SECONDS: ${{ inputs.observe_timeout_seconds }} + POSTMASTER_WA_EVIDENCE_PATH: whatsapp-v990-controlled-evidence.json + run: python scripts/whatsapp_v990_controlled_interop.py + + - name: Upload non-secret interoperability evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: whatsapp-v990-controlled-evidence + path: whatsapp-v990-controlled-evidence.json + if-no-files-found: ignore + retention-days: 1 + + - name: State acceptance boundary + if: always() + run: | + echo "This workflow never changes source acceptance flags automatically." + echo "A release-qualifying run must leave every full-profile input enabled and must end with" + echo "controlled_account_interop_observed=true in the non-secret evidence artifact." + echo "A successful run must be reviewed before protocol_interop_verified, signal_multidevice_verified," + echo "or controlled_account_interop_verified can change in source." + echo "No QR refs, auth database bytes, auth key, JIDs, private keys, session secrets or media payloads" + echo "are written to the evidence artifact." diff --git a/.github/workflows/whatsapp-v990-offline-validation.yml b/.github/workflows/whatsapp-v990-offline-validation.yml new file mode 100644 index 00000000..811eda69 --- /dev/null +++ b/.github/workflows/whatsapp-v990-offline-validation.yml @@ -0,0 +1,219 @@ +name: WhatsApp v9.9 offline validation + +on: + pull_request: + branches: + - main + paths: + - 'src/postmaster/whatsapp_v990/**' + - 'src/postmaster/whatsapp_runtime_v990.py' + - 'src/postmaster/webgui_whatsapp_v990.py' + - 'tests/test_whatsapp_*_v990.py' + - 'scripts/whatsapp_v990_network_smoke.py' + - 'scripts/whatsapp_v990_noise_server_hello_probe.py' + - 'scripts/whatsapp_v990_noise_registration_probe.py' + - 'scripts/whatsapp_v990_pair_device_probe.py' + - 'scripts/whatsapp_v990_controlled_interop.py' + - '.github/workflows/whatsapp-v990-offline-validation.yml' + - '.github/workflows/whatsapp-v990-controlled-interop.yml' + - '.github/workflows/stable-release.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: whatsapp-v990-offline-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + offline-protocol: + name: WhatsApp v9.9 offline protocol/runtime + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install repository runtime dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + + - name: Compile WhatsApp candidate + run: | + python -m compileall -q \ + src/postmaster/whatsapp_v990 \ + src/postmaster/whatsapp_runtime_v990.py \ + src/postmaster/webgui_whatsapp_v990.py \ + scripts/whatsapp_v990_controlled_interop.py \ + tests + + - name: Run WhatsApp offline regression suite + env: + PYTHONPATH: src + run: | + python -W error::ResourceWarning -m unittest discover -s tests -p 'test_whatsapp_*_v990.py' -v + + - name: Guard controlled-account release boundary + run: | + python - <<'PY' + from pathlib import Path + + runtime = Path('src/postmaster/whatsapp_runtime_v990.py').read_text(encoding='utf-8') + harness = Path('scripts/whatsapp_v990_controlled_interop.py').read_text(encoding='utf-8') + workflow = Path('.github/workflows/whatsapp-v990-controlled-interop.yml').read_text(encoding='utf-8') + stable_release = Path('.github/workflows/stable-release.yml').read_text(encoding='utf-8') + + expected_false = ( + '"protocol_interop_verified": False', + '"signal_multidevice_verified": False', + '"controlled_account_interop_verified": False', + ) + missing = [item for item in expected_false if item not in runtime] + assert not missing, ( + 'Live acceptance flags must stay false until controlled-account evidence exists: ' + f'{missing}' + ) + + required_harness_checks = ( + 'POSTMASTER_WA_TEST_GROUP_JID', + 'group_sender_key_interop_observed', + 'media_interop_observed', + 'receipt_policy_observed', + 'inbound_group_media_stored_file_observed', + 'controlled_account_interop_observed', + ) + missing = [item for item in required_harness_checks if item not in harness] + assert not missing, f'Controlled interop harness lost release-critical checks: {missing}' + + required_workflow_checks = ( + 'environment: whatsapp-v990-controlled', + 'POSTMASTER_WA_TEST_GROUP_JID', + 'require_group_echo', + 'require_inbound_media', + 'controlled_account_interop_observed=true', + ) + missing = [item for item in required_workflow_checks if item not in workflow] + assert not missing, f'Controlled interop workflow lost release-critical gates: {missing}' + + release_interlock = ( + "version == '9.9.0'", + '"protocol_interop_verified": True', + '"signal_multidevice_verified": True', + '"controlled_account_interop_verified": True', + 'Refusing stable v9.9.0 release', + ) + missing = [item for item in release_interlock if item not in stable_release] + assert not missing, f'Stable-release WhatsApp interlock is incomplete: {missing}' + + print( + 'Offline validation passed; implementation can advance without an account, ' + 'but stable acceptance remains controlled-account gated.' + ) + PY + + network-transport-smoke: + name: WhatsApp WebSocket unauthenticated transport smoke + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Probe current WhatsApp WebSocket endpoint + run: python scripts/whatsapp_v990_network_smoke.py + + - name: State acceptance boundary + if: always() + run: | + echo "This job validates only DNS/TCP/TLS/RFC6455 reachability." + echo "It does not validate QR pairing, Noise authentication, Signal multi-device, account interoperability, messages, media, groups, or receipts." + + live-noise-server-hello: + name: WhatsApp live Noise ServerHello probe + runs-on: ubuntu-latest + timeout-minutes: 3 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install repository crypto runtime and CI-only WebSocket driver + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + python -m pip install 'websockets>=15,<16' + + - name: Decrypt live Noise ServerHello and verify certificate chain + env: + PYTHONPATH: src + run: python scripts/whatsapp_v990_noise_server_hello_probe.py + + - name: State acceptance boundary + if: always() + run: | + echo "This job stops before ClientFinish/ClientPayload." + echo "No QR/account/device credentials or authenticated WhatsApp actions are exercised." + echo "Passing does not set protocol_interop_verified, signal_multidevice_verified, or controlled_account_interop_verified." + + live-registration-pair-device: + name: WhatsApp live ClientFinish and pair-device probe + runs-on: ubuntu-latest + timeout-minutes: 3 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install repository runtime dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + + - name: Complete throw-away registration and receive encrypted pair-device + env: + PYTHONPATH: src + run: python scripts/whatsapp_v990_pair_device_probe.py + + - name: State acceptance boundary + if: always() + run: | + echo "This job validates ClientFinish, current ClientPayload, encrypted transport and WABinary pair-device refs." + echo "QR refs are never printed or persisted and no account is paired." + echo "Passing still does not set signal_multidevice_verified or controlled_account_interop_verified." + + + live-noise-client-finish: + name: WhatsApp live Noise ClientFinish registration probe + runs-on: ubuntu-latest + timeout-minutes: 3 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install repository crypto runtime and CI-only WebSocket driver + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + python -m pip install 'websockets>=15,<16' + + - name: Send live ClientFinish and authenticate first transport response + env: + PYTHONPATH: src + run: python scripts/whatsapp_v990_noise_registration_probe.py + + - name: State acceptance boundary + if: always() + run: | + echo "This job uses fresh one-run Noise/Signal registration material and stops before QR scan/account pairing." + echo "Passing validates ClientFinish/registration transport compatibility only." + echo "It does not set protocol_interop_verified, signal_multidevice_verified, or controlled_account_interop_verified." diff --git a/docs/whatsapp-v990-release-gate.md b/docs/whatsapp-v990-release-gate.md new file mode 100644 index 00000000..9c84d195 --- /dev/null +++ b/docs/whatsapp-v990-release-gate.md @@ -0,0 +1,109 @@ +# WhatsApp v9.9 release gate + +## Current state + +The clean-room WhatsApp v9.9 implementation is code-complete for the planned Postmaster surface: + +- explicit QR pairing and encrypted auth persistence; +- authenticated reconnect over the current WhatsApp Web companion transport; +- direct multi-device text send/receive; +- encrypted direct media upload/send and inbound media download; +- canonical Postmaster Stored File handoff for media; +- participating group listing; +- Signal SenderKey v3 group text send/receive; +- group media send and inbound group media Stored File handoff; +- remote receipt observation; +- asymmetric read-receipt policy: local reads do not emit receipts, explicit reply actions may; +- local WebGUI pairing QR rendering without remote QR services. + +Automatic CI also performs live unauthenticated/current-protocol probes through ClientFinish and +encrypted pair-device traffic without pairing an account. + +Implementation completeness is **not** the same as controlled-account interoperability acceptance. +The stable v9.9 release remains gated until the manual controlled-account workflow passes. + +## No personal WhatsApp account is required + +Do not use a maintainer's personal WhatsApp account for release acceptance. + +Use dedicated test identities. The full gate is designed for: + +1. one dedicated WhatsApp account that Postmaster pairs as the companion under test; +2. one separate controlled peer WhatsApp account; +3. one test group containing both controlled accounts. + +The peer can be operated manually or by a separate controlled test device. The release gate needs a +distinct peer because it verifies remote multi-device fan-out, remote receipts and inbound direct/group +decryption rather than only self-device synchronization. + +WhatsApp Web/linked-device operation requires an active WhatsApp account on a primary phone. There is +no repository fallback that bypasses this requirement, and the release process must not substitute +unofficial number-verification or disposable-account services. + +## GitHub Environment + +Create a protected GitHub Actions environment named: + +`whatsapp-v990-controlled` + +Configure these environment secrets: + +- `POSTMASTER_WA_AUTH_DB_B64`: Base64 of a previously paired controlled auth SQLite database; +- `POSTMASTER_WA_AUTH_KEY_B64`: Base64 of the corresponding 32-byte auth-store key; +- `POSTMASTER_WA_TEST_JID`: the separate controlled peer JID; +- `POSTMASTER_WA_TEST_GROUP_JID`: a controlled `g.us` group containing the test identities. + +The workflow never prints those values. Its evidence artifact contains only counts and booleans, never +JIDs, QR refs, auth bytes, private keys, session material or media payloads. + +## Release-qualifying workflow + +Run: + +`.github/workflows/whatsapp-v990-controlled-interop.yml` + +For a release-qualifying run, leave all full-profile inputs enabled: + +- direct inbound echo required; +- group inbound echo required; +- inbound group media required; +- remote receipt required; +- media send enabled. + +The controlled peer follows the in-band probe messages. In particular, it echoes the direct/group +tokens and sends one small media attachment to the controlled group with the requested token in its +caption. + +The run is release-qualifying only when the non-secret evidence ends with all of these true: + +- `protocol_interop_observed`; +- `signal_multidevice_observed`; +- `group_sender_key_interop_observed`; +- `media_interop_observed`; +- `receipt_policy_observed`; +- `controlled_account_interop_observed`. + +A diagnostic run with any full-profile requirement disabled is useful for debugging but must not +qualify a stable release. + +## Promotion sequence + +Before the controlled-account gate passes: + +- keep `protocol_interop_verified = false`; +- keep `signal_multidevice_verified = false`; +- keep `controlled_account_interop_verified = false`; +- keep the project VERSION on the current stable release; +- do not create the v9.9.0 stable tag/release. + +After a reviewed full-profile pass: + +1. set the three acceptance flags to true; +2. run the complete repository regression suite again; +3. update VERSION/CHANGELOG/release metadata to v9.9.0; +4. verify the public single-YAML bootstrap and dependency install from a clean environment; +5. merge the release branch; +6. create the v9.9.0 tag and stable GitHub release. + +This separation allows development and CI hardening to reach release-candidate quality without +requiring any maintainer's personal WhatsApp identity. diff --git a/requirements.txt b/requirements.txt index d89d4e25..e25bc490 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ huggingface-hub>=0.24,<2 tokenizers>=0.20,<1 mistune>=3,<4 bleach>=6.2,<7 +qrcode==8.2 diff --git a/scripts/whatsapp_v990_controlled_interop.py b/scripts/whatsapp_v990_controlled_interop.py new file mode 100644 index 00000000..5e0a30f4 --- /dev/null +++ b/scripts/whatsapp_v990_controlled_interop.py @@ -0,0 +1,567 @@ +from __future__ import annotations + +"""Controlled-account WhatsApp v9.9 interoperability harness for GitHub Actions. + +Secrets are supplied only through the workflow environment. The script never prints or writes +QR refs, JIDs, private keys, auth database bytes, auth key material, downloaded media bytes, or +stored-file payloads to the evidence artifact. + +The full acceptance profile restores a previously paired controlled test account and exercises: +- authenticated reconnect and pre-key maintenance, +- participating-group listing, +- direct multi-device text and media send, +- group SenderKey text and media send, +- remote receipt observation, +- inbound direct text decrypt, +- inbound group SenderKey text decrypt, +- inbound group media decrypt/download and canonical Stored File handoff. + +The test peer is instructed in-band using random one-run tokens. This harness never changes source +acceptance flags automatically; a successful run is evidence for a human-reviewed release gate. +""" + +import asyncio +import base64 +from dataclasses import asdict, dataclass +import json +import os +from pathlib import Path +import secrets +import stat +import tempfile +import time +from typing import Any + +from postmaster.whatsapp_v990.adapter import CurrentProtocolAdapter +from postmaster.whatsapp_v990.jid import parse_jid +from postmaster.whatsapp_v990.service import WhatsAppEventStore +from postmaster.whatsapp_v990.store import EncryptedAuthStore + + +class HarnessError(RuntimeError): + pass + + +@dataclass +class Evidence: + reconnect_ok: bool = False + prekey_server_count: int | None = None + + group_listing_ok: bool = False + group_count: int | None = None + target_group_visible: bool = False + + direct_text_send_ok: bool = False + direct_text_server_ack: bool = False + direct_text_device_fanout: int | None = None + direct_text_remote_device_targets: int | None = None + direct_text_own_device_targets: int | None = None + direct_text_used_prekey_message: bool | None = None + direct_text_read_receipt_emitted: bool = False + + direct_media_send_ok: bool = False + direct_media_server_ack: bool = False + direct_media_device_fanout: int | None = None + direct_media_read_receipt_emitted: bool = False + + group_text_send_ok: bool = False + group_text_server_ack: bool = False + group_text_device_fanout: int | None = None + group_text_sender_key_recipients: int | None = None + group_text_read_receipt_emitted: bool = False + + group_media_send_ok: bool = False + group_media_server_ack: bool = False + group_media_device_fanout: int | None = None + group_media_sender_key_recipients: int | None = None + group_media_read_receipt_emitted: bool = False + + remote_receipt_observed: bool = False + inbound_direct_echo_observed: bool = False + inbound_group_echo_observed: bool = False + inbound_group_media_observed: bool = False + inbound_group_media_stored_file_observed: bool = False + inbound_stored_file_count: int = 0 + + protocol_interop_observed: bool = False + signal_multidevice_observed: bool = False + group_sender_key_interop_observed: bool = False + media_interop_observed: bool = False + receipt_policy_observed: bool = False + controlled_account_interop_observed: bool = False + + +class HarnessFileStore: + """In-memory Stored File boundary for a one-run controlled probe. + + Outbound fixture bytes and inbound downloaded bytes stay in process memory only. Evidence + contains counts/booleans, never media bytes, filenames supplied by the peer, or file IDs. + """ + + def __init__(self) -> None: + self._files: dict[str, tuple[dict[str, Any], bytes]] = { + "controlled-interop-document": ( + {"id": "controlled-interop-document", "filename": "postmaster-v990-controlled.txt", "media_type": "text/plain"}, + b"Postmaster MCP v9.9 controlled WhatsApp media interoperability probe.\n", + ) + } + self.inbound_saved = 0 + + def raw_bytes(self, file_id: str): + if file_id not in self._files: + raise KeyError(file_id) + info, data = self._files[file_id] + return dict(info), bytes(data) + + def save_bytes( + self, + *, + owner_id: str, + filename: str, + data: bytes, + project_id: str | None = None, + media_type: str | None = None, + description: str = "", + tags: list[str] | None = None, + file_id: str | None = None, + ) -> dict[str, Any]: + fid = file_id or ("controlled-inbound-" + secrets.token_hex(8)) + info = { + "id": fid, + "owner_id": str(owner_id), + "project_id": str(project_id) if project_id else None, + "filename": str(filename), + "media_type": str(media_type or "application/octet-stream"), + "size_bytes": len(bytes(data)), + "description": str(description or ""), + "tags": list(tags or []), + } + self._files[fid] = (dict(info), bytes(data)) + self.inbound_saved += 1 + return dict(info) + + +def _required(name: str) -> str: + value = str(os.environ.get(name) or "").strip() + if not value: + raise HarnessError(f"Required controlled-interop environment value is missing: {name}") + return value + + +def _bool(name: str, default: bool) -> bool: + value = os.environ.get(name) + if value is None: + return default + return str(value).strip().lower() in {"1", "true", "yes", "on"} + + +def _decode_secret(name: str) -> bytes: + value = _required(name) + try: + raw = base64.b64decode(value, validate=True) + except Exception as exc: + raise HarnessError(f"{name} is not valid Base64") from exc + if not raw: + raise HarnessError(f"{name} decoded to empty bytes") + return raw + + +def _write_secret(path: Path, data: bytes) -> None: + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(data) + except Exception: + try: + path.unlink(missing_ok=True) + except OSError: + pass + raise + if stat.S_IMODE(path.stat().st_mode) & 0o077: + raise HarnessError("Controlled auth secret file permissions are broader than 0600") + + +async def _wait_for_receipt(events: WhatsAppEventStore, message_id: str, timeout: float) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + for row in events.list_receipts(limit=500): + if ( + str(row.get("message_id") or "") == message_id + and str(row.get("source") or "") == "remote" + ): + return True + await asyncio.sleep(1.0) + return False + + +async def _wait_for_message( + events: WhatsAppEventStore, + *, + jid: str, + token: str, + kind: str, + timeout: float, +) -> dict[str, Any] | None: + wanted_jid = str(parse_jid(jid)) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + for row in events.list_messages(jid=wanted_jid, limit=500): + if str(row.get("direction") or "") != "in": + continue + if str(row.get("kind") or "") != kind: + continue + if token not in str(row.get("text") or ""): + continue + return row + await asyncio.sleep(1.0) + return None + + +def _int_or_none(value: Any) -> int | None: + return int(value) if value is not None else None + + +def _assert_no_local_read_receipt(label: str, result: dict[str, Any]) -> None: + if bool(result.get("read_receipt_emitted")): + raise HarnessError(f"{label} unexpectedly emitted a local WhatsApp read receipt") + + +async def main() -> int: + auth_db_bytes = _decode_secret("POSTMASTER_WA_AUTH_DB_B64") + auth_key_bytes = _decode_secret("POSTMASTER_WA_AUTH_KEY_B64") + if len(auth_key_bytes) != 32: + raise HarnessError("POSTMASTER_WA_AUTH_KEY_B64 must decode to exactly 32 bytes") + + test_jid = str(parse_jid(_required("POSTMASTER_WA_TEST_JID")).normalized_user()) + test_group_jid = str(parse_jid(_required("POSTMASTER_WA_TEST_GROUP_JID"))) + if not parse_jid(test_group_jid).is_group: + raise HarnessError("POSTMASTER_WA_TEST_GROUP_JID must be a g.us group JID") + + require_receipt = _bool("POSTMASTER_WA_REQUIRE_REMOTE_RECEIPT", True) + require_direct_echo = _bool("POSTMASTER_WA_REQUIRE_INBOUND_ECHO", True) + require_group_echo = _bool("POSTMASTER_WA_REQUIRE_GROUP_ECHO", True) + require_inbound_media = _bool("POSTMASTER_WA_REQUIRE_INBOUND_MEDIA", True) + run_media = _bool("POSTMASTER_WA_RUN_MEDIA", True) + timeout = float(os.environ.get("POSTMASTER_WA_OBSERVE_TIMEOUT_SECONDS") or "120") + timeout = min(max(timeout, 10.0), 300.0) + evidence_path = Path( + os.environ.get("POSTMASTER_WA_EVIDENCE_PATH") + or "whatsapp-v990-controlled-evidence.json" + ) + + evidence = Evidence() + with tempfile.TemporaryDirectory(prefix="postmaster-wa-controlled-") as td: + root = Path(td) + auth_db = root / "auth.db" + auth_key = root / "auth.key" + event_db = root / "events.db" + _write_secret(auth_db, auth_db_bytes) + _write_secret(auth_key, auth_key_bytes) + + auth = EncryptedAuthStore(str(auth_db), key_path=str(auth_key)) + events = WhatsAppEventStore(str(event_db)) + files = HarnessFileStore() + adapter = CurrentProtocolAdapter( + auth, + on_message=lambda **kwargs: events.record_message(**kwargs), + on_receipt=lambda **kwargs: events.record_receipt(**kwargs), + file_store=files, + file_owner_id="controlled-interop", + ) + + try: + connected = dict(await adapter.reconnect()) + evidence.reconnect_ok = bool(connected.get("connected")) + evidence.prekey_server_count = _int_or_none(connected.get("server_pre_key_count")) + if not evidence.reconnect_ok: + raise HarnessError("Controlled WhatsApp session did not reconnect") + + groups = [dict(item) for item in await adapter.list_groups()] + evidence.group_listing_ok = True + evidence.group_count = len(groups) + evidence.target_group_visible = any( + str(parse_jid(str(item.get("id") or ""))) == test_group_jid + for item in groups + if item.get("id") + ) + if not evidence.target_group_visible: + raise HarnessError("Controlled test group is not visible to the paired companion") + + direct_token = "postmaster-direct-" + secrets.token_hex(12) + direct = dict( + await adapter.send_text( + jid=test_jid, + text=f"ECHO_DIRECT:{direct_token}", + ) + ) + evidence.direct_text_send_ok = bool(direct.get("ok")) + evidence.direct_text_server_ack = bool(direct.get("server_ack")) + evidence.direct_text_device_fanout = _int_or_none(direct.get("device_fanout")) + evidence.direct_text_remote_device_targets = _int_or_none( + direct.get("remote_device_targets") + ) + evidence.direct_text_own_device_targets = _int_or_none( + direct.get("own_device_targets") + ) + evidence.direct_text_used_prekey_message = bool(direct.get("used_prekey_message")) + evidence.direct_text_read_receipt_emitted = bool( + direct.get("read_receipt_emitted") + ) + _assert_no_local_read_receipt("Controlled direct text send", direct) + direct_message_id = str(direct.get("message_id") or "") + if ( + not evidence.direct_text_send_ok + or not evidence.direct_text_server_ack + or not direct_message_id + ): + raise HarnessError( + "Controlled direct-text send did not receive a clean server ack" + ) + + if run_media: + direct_media = dict( + await adapter.send_media( + jid=test_jid, + stored_file_id="controlled-interop-document", + caption="Postmaster v9.9 controlled direct media probe", + ) + ) + evidence.direct_media_send_ok = bool(direct_media.get("ok")) + evidence.direct_media_server_ack = bool(direct_media.get("server_ack")) + evidence.direct_media_device_fanout = _int_or_none( + direct_media.get("device_fanout") + ) + evidence.direct_media_read_receipt_emitted = bool( + direct_media.get("read_receipt_emitted") + ) + _assert_no_local_read_receipt( + "Controlled direct media send", + direct_media, + ) + if ( + not evidence.direct_media_send_ok + or not evidence.direct_media_server_ack + ): + raise HarnessError( + "Controlled direct-media send did not receive a clean server ack" + ) + + group_token = "postmaster-group-" + secrets.token_hex(12) + group_text = dict( + await adapter.send_text( + jid=test_group_jid, + text=f"ECHO_GROUP:{group_token}", + ) + ) + evidence.group_text_send_ok = bool(group_text.get("ok")) + evidence.group_text_server_ack = bool(group_text.get("server_ack")) + evidence.group_text_device_fanout = _int_or_none( + group_text.get("device_fanout") + ) + evidence.group_text_sender_key_recipients = _int_or_none( + group_text.get("sender_key_recipients") + ) + evidence.group_text_read_receipt_emitted = bool( + group_text.get("read_receipt_emitted") + ) + _assert_no_local_read_receipt("Controlled group text send", group_text) + if ( + not evidence.group_text_send_ok + or not evidence.group_text_server_ack + ): + raise HarnessError( + "Controlled group-text send did not receive a clean server ack" + ) + + media_token = "postmaster-media-" + secrets.token_hex(12) + await adapter.send_text( + jid=test_group_jid, + text=( + "CONTROLLED_MEDIA_REQUEST:" + + media_token + + " — send one small media attachment to this group with this exact token in its caption." + ), + ) + + if run_media: + group_media = dict( + await adapter.send_media( + jid=test_group_jid, + stored_file_id="controlled-interop-document", + caption="Postmaster v9.9 controlled group media probe", + ) + ) + evidence.group_media_send_ok = bool(group_media.get("ok")) + evidence.group_media_server_ack = bool(group_media.get("server_ack")) + evidence.group_media_device_fanout = _int_or_none( + group_media.get("device_fanout") + ) + evidence.group_media_sender_key_recipients = _int_or_none( + group_media.get("sender_key_recipients") + ) + evidence.group_media_read_receipt_emitted = bool( + group_media.get("read_receipt_emitted") + ) + _assert_no_local_read_receipt( + "Controlled group media send", + group_media, + ) + if ( + not evidence.group_media_send_ok + or not evidence.group_media_server_ack + ): + raise HarnessError( + "Controlled group-media send did not receive a clean server ack" + ) + + receipt_coro = ( + _wait_for_receipt(events, direct_message_id, timeout) + if require_receipt + else asyncio.sleep(0, result=False) + ) + direct_echo_coro = ( + _wait_for_message( + events, + jid=test_jid, + token=direct_token, + kind="text", + timeout=timeout, + ) + if require_direct_echo + else asyncio.sleep(0, result=None) + ) + group_echo_coro = ( + _wait_for_message( + events, + jid=test_group_jid, + token=group_token, + kind="text", + timeout=timeout, + ) + if require_group_echo + else asyncio.sleep(0, result=None) + ) + group_media_coro = ( + _wait_for_message( + events, + jid=test_group_jid, + token=media_token, + kind="media", + timeout=timeout, + ) + if require_inbound_media + else asyncio.sleep(0, result=None) + ) + + ( + receipt_observed, + direct_echo, + group_echo, + group_media_in, + ) = await asyncio.gather( + receipt_coro, + direct_echo_coro, + group_echo_coro, + group_media_coro, + ) + evidence.remote_receipt_observed = bool(receipt_observed) + evidence.inbound_direct_echo_observed = direct_echo is not None + evidence.inbound_group_echo_observed = group_echo is not None + evidence.inbound_group_media_observed = group_media_in is not None + evidence.inbound_group_media_stored_file_observed = bool( + group_media_in and group_media_in.get("stored_file_id") + ) + evidence.inbound_stored_file_count = files.inbound_saved + + evidence.receipt_policy_observed = bool( + not evidence.direct_text_read_receipt_emitted + and not evidence.direct_media_read_receipt_emitted + and not evidence.group_text_read_receipt_emitted + and not evidence.group_media_read_receipt_emitted + ) + evidence.protocol_interop_observed = bool( + evidence.reconnect_ok + and evidence.group_listing_ok + and evidence.target_group_visible + and evidence.direct_text_server_ack + and evidence.group_text_server_ack + and (not run_media or evidence.direct_media_server_ack) + and (not run_media or evidence.group_media_server_ack) + ) + evidence.signal_multidevice_observed = bool( + evidence.direct_text_server_ack + and (evidence.direct_text_device_fanout or 0) > 0 + and (evidence.direct_text_remote_device_targets or 0) > 0 + and evidence.inbound_direct_echo_observed + ) + evidence.group_sender_key_interop_observed = bool( + evidence.group_text_server_ack + and (evidence.group_text_device_fanout or 0) > 0 + and evidence.inbound_group_echo_observed + ) + evidence.media_interop_observed = bool( + run_media + and evidence.direct_media_server_ack + and evidence.group_media_server_ack + and evidence.inbound_group_media_observed + and evidence.inbound_group_media_stored_file_observed + and evidence.inbound_stored_file_count > 0 + ) + evidence.controlled_account_interop_observed = bool( + evidence.protocol_interop_observed + and evidence.signal_multidevice_observed + and evidence.group_sender_key_interop_observed + and evidence.media_interop_observed + and evidence.receipt_policy_observed + and evidence.remote_receipt_observed + ) + + if require_receipt and not evidence.remote_receipt_observed: + raise HarnessError( + "No remote receipt for the controlled direct text was observed before timeout" + ) + if require_direct_echo and not evidence.inbound_direct_echo_observed: + raise HarnessError( + "No inbound controlled direct echo was observed before timeout" + ) + if require_group_echo and not evidence.inbound_group_echo_observed: + raise HarnessError( + "No inbound controlled group echo was observed before timeout" + ) + if require_inbound_media and not evidence.inbound_group_media_stored_file_observed: + raise HarnessError( + "No inbound controlled group media was decrypted and stored before timeout" + ) + if not evidence.receipt_policy_observed: + raise HarnessError( + "A non-reply controlled send unexpectedly emitted a local read receipt" + ) + if not evidence.controlled_account_interop_observed: + raise HarnessError( + "Controlled-account full acceptance conditions were not all satisfied" + ) + finally: + await adapter.close() + + evidence_path.write_text( + json.dumps(asdict(evidence), sort_keys=True, indent=2) + "\n", + encoding="utf-8", + ) + print( + "Controlled WhatsApp interoperability evidence written without account identifiers, " + "key material or media payloads." + ) + print(f"protocol_interop_observed={evidence.protocol_interop_observed}") + print(f"signal_multidevice_observed={evidence.signal_multidevice_observed}") + print(f"group_sender_key_interop_observed={evidence.group_sender_key_interop_observed}") + print(f"media_interop_observed={evidence.media_interop_observed}") + print(f"receipt_policy_observed={evidence.receipt_policy_observed}") + print( + "controlled_account_interop_observed=" + f"{evidence.controlled_account_interop_observed}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/scripts/whatsapp_v990_network_smoke.py b/scripts/whatsapp_v990_network_smoke.py new file mode 100644 index 00000000..07302e5e --- /dev/null +++ b/scripts/whatsapp_v990_network_smoke.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +"""Unauthenticated WhatsApp Web transport smoke for CI. + +This proves only DNS/TCP/TLS plus RFC6455 Upgrade reachability for the current clean-room +transport endpoint. It never sends a Noise handshake, QR material, credentials, cookies, +account identifiers, messages, or other authenticated protocol data. +""" + +import base64 +import hashlib +import os +import socket +import ssl +import sys + +HOST = "web.whatsapp.com" +PORT = 443 +PATH = "/ws/chat" +ORIGIN = "https://web.whatsapp.com" +GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" +MAX_HEADER_BYTES = 64 * 1024 + + +def _headers(raw: bytes) -> tuple[str, dict[str, str]]: + head = raw.decode("iso-8859-1", errors="replace") + lines = head.split("\r\n") + status = lines[0] if lines else "" + headers: dict[str, str] = {} + for line in lines[1:]: + if ":" not in line: + continue + key, value = line.split(":", 1) + headers[key.strip().lower()] = value.strip() + return status, headers + + +def main() -> int: + key = base64.b64encode(os.urandom(16)).decode("ascii") + request = ( + f"GET {PATH} HTTP/1.1\r\n" + f"Host: {HOST}\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + f"Sec-WebSocket-Key: {key}\r\n" + "Sec-WebSocket-Version: 13\r\n" + f"Origin: {ORIGIN}\r\n" + "User-Agent: Postmaster-v9.9-CI-transport-smoke/1.0\r\n" + "\r\n" + ).encode("ascii") + + context = ssl.create_default_context() + with socket.create_connection((HOST, PORT), timeout=15) as tcp: + with context.wrap_socket(tcp, server_hostname=HOST) as tls: + tls.settimeout(15) + tls.sendall(request) + response = bytearray() + while b"\r\n\r\n" not in response: + chunk = tls.recv(4096) + if not chunk: + break + response.extend(chunk) + if len(response) > MAX_HEADER_BYTES: + raise RuntimeError("WebSocket response headers exceeded safety bound") + + header_block = bytes(response).split(b"\r\n\r\n", 1)[0] + status, headers = _headers(header_block) + if not status.startswith("HTTP/1.1 101"): + print(f"WhatsApp WebSocket transport smoke failed: {status or 'no HTTP status'}", file=sys.stderr) + return 2 + + upgrade = headers.get("upgrade", "").lower() + connection = headers.get("connection", "").lower() + expected_accept = base64.b64encode( + hashlib.sha1((key + GUID).encode("ascii")).digest() + ).decode("ascii") + actual_accept = headers.get("sec-websocket-accept", "") + + if upgrade != "websocket" or "upgrade" not in connection: + print("WhatsApp endpoint did not return a valid WebSocket Upgrade response", file=sys.stderr) + return 3 + if actual_accept != expected_accept: + print("WhatsApp endpoint returned an invalid Sec-WebSocket-Accept", file=sys.stderr) + return 4 + + peer = tls.getpeercert() + print("WhatsApp WebSocket transport reachable: HTTP 101") + print(f"TLS version: {tls.version()}") + print(f"Certificate subject present: {bool(peer.get('subject'))}") + print("Authentication/protocol/account interoperability: NOT TESTED") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/whatsapp_v990_noise_registration_probe.py b/scripts/whatsapp_v990_noise_registration_probe.py new file mode 100644 index 00000000..b96c4028 --- /dev/null +++ b/scripts/whatsapp_v990_noise_registration_probe.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +"""Live, unauthenticated WhatsApp Noise ClientFinish/registration probe. + +The probe uses fresh ephemeral Noise + Signal identity material on every run. It sends a +registration-shaped ClientPayload through the current Noise XX handshake and verifies that the +first post-handshake transport frame can be authenticated/decrypted with the derived transport +keys. It deliberately stops before QR display/scan and never pairs an account. + +Passing is evidence for: +- current WebSocket endpoint/origin reachability, +- Noise XX ClientHello/ServerHello/ClientFinish compatibility, +- pinned server certificate-chain verification, +- current registration ClientPayload wire acceptance far enough to enter transport mode. + +Passing is NOT evidence for account pairing, Signal multi-device lifecycle, messaging, media, +groups, receipts, or controlled-account interoperability. +""" + +import asyncio +import hashlib + +from postmaster.whatsapp_v990.client_payload import RegistrationKeys, build_registration_payload +from postmaster.whatsapp_v990.crypto import ( + NOISE_WA_HEADER, + WhatsAppNoiseXX, + frame_noise_payload, + generate_curve_keypair, + split_noise_frames, +) +from postmaster.whatsapp_v990.handshake import decode_handshake, encode_client_finish, encode_client_hello +from postmaster.whatsapp_v990.signal_keys import generate_registration_id, generate_signed_pre_key +from postmaster.whatsapp_v990.websocket_driver import open_whatsapp_websocket + + +async def _recv_frames(ws, *, timeout: float = 15.0) -> list[bytes]: + incoming = await asyncio.wait_for(ws.recv(), timeout=timeout) + if isinstance(incoming, str): + raise RuntimeError("WhatsApp server returned text during binary Noise session") + frames, rest = split_noise_frames(bytes(incoming)) + if rest: + raise RuntimeError(f"partial trailing Noise frame received: {len(rest)} bytes") + if not frames: + raise RuntimeError("WhatsApp returned no complete Noise frame") + return frames + + +async def main() -> int: + ephemeral = generate_curve_keypair() + noise_static = generate_curve_keypair() + identity = generate_curve_keypair() + signed_pre_key = generate_signed_pre_key(identity, 1) + registration = RegistrationKeys( + registration_id=generate_registration_id(), + identity_public_key=identity.public, + signed_pre_key=signed_pre_key, + ) + payload = build_registration_payload(registration) + + ws = await open_whatsapp_websocket() + try: + await ws.send(frame_noise_payload(encode_client_hello(ephemeral.public), intro=NOISE_WA_HEADER)) + server_frames = await _recv_frames(ws) + if len(server_frames) != 1: + raise RuntimeError(f"expected one ServerHello frame, received {len(server_frames)}") + + handshake = decode_handshake(server_frames[0]) + server = handshake.server_hello + if server is None: + raise RuntimeError("WhatsApp did not return ServerHello") + + noise = WhatsAppNoiseXX(ephemeral) + encrypted_client_static, _cert_chain = noise.process_server_hello( + server_ephemeral=server.ephemeral, + encrypted_static=server.static, + encrypted_payload=server.payload, + noise_static=noise_static, + ) + encrypted_payload = noise.encrypt(payload) + finish = encode_client_finish(encrypted_client_static, encrypted_payload) + await ws.send(frame_noise_payload(finish)) + + transport = noise.finish() + post_frames = await _recv_frames(ws) + plaintext = [transport.decrypt(frame) for frame in post_frames] + if not plaintext or not any(item for item in plaintext): + raise RuntimeError("post-ClientFinish transport response decrypted to empty payloads") + + total = sum(len(item) for item in plaintext) + digest = hashlib.sha256(b"".join(plaintext)).hexdigest() + print("WhatsApp live Noise ClientFinish accepted into authenticated transport framing") + print(f"Registration ClientPayload bytes: {len(payload)}") + print(f"Post-handshake transport frames: {len(plaintext)}") + print(f"Post-handshake plaintext bytes: {total}") + print(f"Post-handshake plaintext SHA256: {digest}") + print("QR/account pairing: NOT PERFORMED") + print("Signal multi-device/message/media/group/receipt interoperability: NOT TESTED") + return 0 + finally: + await ws.close() + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/scripts/whatsapp_v990_noise_server_hello_probe.py b/scripts/whatsapp_v990_noise_server_hello_probe.py new file mode 100644 index 00000000..0f381478 --- /dev/null +++ b/scripts/whatsapp_v990_noise_server_hello_probe.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +"""Live, unauthenticated WhatsApp Noise server-hello probe. + +This stops before ClientFinish/ClientPayload. It uses only ephemeral/random client keys and +validates the server Noise certificate chain. No QR, account, cookie, message or credential +is sent. Passing this probe is evidence for the initial Noise XX server-hello path only. +""" + +import asyncio +import time + +from postmaster.whatsapp_v990.cert import verify_noise_certificate_chain +from postmaster.whatsapp_v990.crypto import ( + NOISE_WA_HEADER, + WhatsAppNoiseXX, + generate_curve_keypair, + split_noise_frames, + frame_noise_payload, +) +from postmaster.whatsapp_v990.handshake import decode_handshake, encode_client_hello +from postmaster.whatsapp_v990.proto import ProtoField, decode_fields +from postmaster.whatsapp_v990.websocket_driver import open_whatsapp_websocket + + +async def main() -> int: + ephemeral = generate_curve_keypair() + noise_static = generate_curve_keypair() + noise = WhatsAppNoiseXX(ephemeral) + + ws = await open_whatsapp_websocket() + try: + client_hello = encode_client_hello(ephemeral.public) + await ws.send(frame_noise_payload(client_hello, intro=NOISE_WA_HEADER)) + + incoming = await asyncio.wait_for(ws.recv(), timeout=15) + if isinstance(incoming, str): + raise RuntimeError("WhatsApp server returned text during Noise handshake") + frames, rest = split_noise_frames(bytes(incoming)) + if rest: + raise RuntimeError(f"partial trailing Noise frame received: {len(rest)} bytes") + if len(frames) != 1: + raise RuntimeError(f"expected one server Noise frame, received {len(frames)}") + + handshake = decode_handshake(frames[0]) + server = handshake.server_hello + if server is None: + raise RuntimeError("WhatsApp did not return ServerHello") + + _encrypted_client_static, cert_chain = noise.process_server_hello( + server_ephemeral=server.ephemeral, + encrypted_static=server.static, + encrypted_payload=server.payload, + noise_static=noise_static, + ) + cert = verify_noise_certificate_chain(cert_chain) + + print("WhatsApp live Noise ServerHello decrypted successfully") + print(f"Server ephemeral bytes: {len(server.ephemeral)}") + print(f"Server certificate chain bytes: {len(cert_chain)}") + print(f"Certificate signatures verified: {cert.chain_signatures_verified}") + print(f"Issuer serial: {cert.intermediate.issuer_serial}") + print("ClientFinish/account authentication: NOT SENT / NOT TESTED") + return 0 + finally: + await ws.close() + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/scripts/whatsapp_v990_pair_device_probe.py b/scripts/whatsapp_v990_pair_device_probe.py new file mode 100644 index 00000000..6442a13d --- /dev/null +++ b/scripts/whatsapp_v990_pair_device_probe.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +"""Live unauthenticated WhatsApp registration -> pair-device probe. + +The probe completes the public Noise XX ClientFinish with a fresh, throw-away registration +bundle and waits for WhatsApp's encrypted WABinary pair-device stanza. It never displays or +persists QR refs, never scans a QR, never authenticates an account, and never sends messages. +""" + +import asyncio +import os + +from postmaster.whatsapp_v990.binary import BinaryNode, BinaryNodeCodec, BinaryNodeError +from postmaster.whatsapp_v990.cert import verify_noise_certificate_chain +from postmaster.whatsapp_v990.client_payload import RegistrationKeys, build_registration_payload +from postmaster.whatsapp_v990.crypto import ( + NOISE_WA_HEADER, + WhatsAppNoiseXX, + frame_noise_payload, + generate_curve_keypair, + split_noise_frames, +) +from postmaster.whatsapp_v990.handshake import decode_handshake, encode_client_finish, encode_client_hello +from postmaster.whatsapp_v990.signal_keys import generate_registration_id, generate_signed_pre_key +from postmaster.whatsapp_v990.tokens import CURRENT_TOKEN_TABLE +from postmaster.whatsapp_v990.websocket_driver import WebSocketDriverConfig, open_whatsapp_websocket + + +async def _recv_frames(ws, pending: bytes, *, timeout: float = 15.0) -> tuple[list[bytes], bytes]: + value = await asyncio.wait_for(ws.recv(), timeout=timeout) + if isinstance(value, str): + raise RuntimeError("WhatsApp returned a text WebSocket frame during binary protocol setup") + return split_noise_frames(pending + bytes(value)) + + +async def main() -> int: + ephemeral = generate_curve_keypair() + noise_static = generate_curve_keypair() + identity = generate_curve_keypair() + signed_pre_key = generate_signed_pre_key(identity, 1) + registration_id = generate_registration_id() + + registration_payload = build_registration_payload( + RegistrationKeys( + registration_id=registration_id, + identity_public_key=identity.public, + signed_pre_key=signed_pre_key, + ) + ) + + noise = WhatsAppNoiseXX(ephemeral) + codec = BinaryNodeCodec(CURRENT_TOKEN_TABLE) + ws = await open_whatsapp_websocket(WebSocketDriverConfig(prefer_native=True)) + pending = b"" + try: + await ws.send(frame_noise_payload(encode_client_hello(ephemeral.public), intro=NOISE_WA_HEADER)) + + frames: list[bytes] = [] + while not frames: + frames, pending = await _recv_frames(ws, pending) + if len(frames) != 1: + raise RuntimeError(f"expected one Noise ServerHello frame, received {len(frames)}") + + hello = decode_handshake(frames[0]).server_hello + if hello is None: + raise RuntimeError("WhatsApp did not return Noise ServerHello") + + encrypted_static, cert_chain = noise.process_server_hello( + server_ephemeral=hello.ephemeral, + encrypted_static=hello.static, + encrypted_payload=hello.payload, + noise_static=noise_static, + ) + cert = verify_noise_certificate_chain(cert_chain) + encrypted_payload = noise.encrypt(registration_payload) + await ws.send(frame_noise_payload(encode_client_finish(encrypted_static, encrypted_payload))) + transport = noise.finish() + + pair_device: BinaryNode | None = None + pair_iq: BinaryNode | None = None + decoded_count = 0 + for _ in range(24): + encrypted_frames, pending = await _recv_frames(ws, pending, timeout=15.0) + for encrypted in encrypted_frames: + plaintext = transport.decrypt(encrypted) + try: + node = codec.decode(plaintext) + except BinaryNodeError as exc: + raise RuntimeError(f"current WABinary token dictionary could not decode live traffic: {exc}") from exc + decoded_count += 1 + candidate = node.child("pair-device") if node.tag == "iq" else None + if candidate is not None: + pair_iq, pair_device = node, candidate + break + if pair_device is not None: + break + + if pair_device is None or pair_iq is None: + raise RuntimeError(f"pair-device stanza not observed after {decoded_count} decrypted WABinary frames") + + refs = pair_device.children("ref") + if not refs: + raise RuntimeError("pair-device stanza contained no QR reference nodes") + if not pair_iq.attrs.get("id"): + raise RuntimeError("pair-device IQ has no stanza id") + + # Acknowledge the stanza exactly as a Web companion is expected to do. This does not + # authenticate or pair an account; it only confirms receipt of the ephemeral QR refs. + ack = BinaryNode( + "iq", + {"to": "s.whatsapp.net", "type": "result", "id": pair_iq.attrs["id"]}, + ) + ack_wire = codec.encode(ack) + await ws.send(frame_noise_payload(transport.encrypt(ack_wire))) + + print("WhatsApp live registration ClientFinish accepted") + print(f"Noise certificate signatures verified: {cert.chain_signatures_verified}") + print(f"Current WABinary token table version: {CURRENT_TOKEN_TABLE.version}") + print(f"Encrypted transport frames decoded before pair-device: {decoded_count}") + print(f"Pair-device QR references received: {len(refs)}") + print("QR reference contents: NOT PRINTED / NOT PERSISTED") + print("Account pairing/authentication: NOT PERFORMED") + return 0 + finally: + await ws.close() + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/src/postmaster/calendar_v990.py b/src/postmaster/calendar_v990.py new file mode 100644 index 00000000..1bdf0279 --- /dev/null +++ b/src/postmaster/calendar_v990.py @@ -0,0 +1,421 @@ +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from email.message import Message +from email.policy import default +from email.parser import BytesParser +from typing import Any, Iterable +from urllib.parse import urlparse + +_CALENDAR_TYPES = { + "text/calendar", + "application/calendar", + "application/ics", + "application/icalendar", + "text/x-vcalendar", +} +_VIDEO_HOSTS = { + "meet.google.com", + "teams.microsoft.com", + "teams.live.com", + "zoom.us", + "www.zoom.us", + "webex.com", + "www.webex.com", + "whereby.com", + "jitsi.org", + "meet.jit.si", +} +_URL_RE = re.compile(r"https?://[^\s<>\"']+", re.I) + + +class CalendarError(ValueError): + pass + + +def _unfold_ical(text: str) -> list[str]: + normalized = text.replace("\r\n", "\n").replace("\r", "\n") + out: list[str] = [] + for line in normalized.split("\n"): + if line.startswith((" ", "\t")) and out: + out[-1] += line[1:] + else: + out.append(line) + return out + + +def _split_property(line: str) -> tuple[str, dict[str, str], str] | None: + if ":" not in line: + return None + left, value = line.split(":", 1) + bits = left.split(";") + name = bits[0].strip().upper() + if not name: + return None + params: dict[str, str] = {} + for bit in bits[1:]: + if "=" in bit: + k, v = bit.split("=", 1) + params[k.strip().upper()] = v.strip().strip('"') + return name, params, value.strip() + + +def _ical_unescape(value: str) -> str: + return ( + value.replace("\\N", "\n") + .replace("\\n", "\n") + .replace("\\,", ",") + .replace("\\;", ";") + .replace("\\\\", "\\") + ) + + +def _ical_escape(value: str) -> str: + return ( + str(value) + .replace("\\", "\\\\") + .replace(";", "\\;") + .replace(",", "\\,") + .replace("\r\n", "\\n") + .replace("\r", "\\n") + .replace("\n", "\\n") + ) + + +def _parse_dt(value: str, params: dict[str, str]) -> dict[str, Any]: + raw = value.strip() + timezone = params.get("TZID") or "" + date_only = params.get("VALUE", "").upper() == "DATE" or bool(re.fullmatch(r"\d{8}", raw)) + dt: datetime | None = None + try: + if date_only: + dt = datetime.strptime(raw[:8], "%Y%m%d").replace(tzinfo=UTC) + elif raw.endswith("Z"): + dt = datetime.strptime(raw, "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC) + elif re.fullmatch(r"\d{8}T\d{6}", raw): + # A TZID value is kept explicitly instead of pretending the local wall time is UTC. + dt = datetime.strptime(raw, "%Y%m%dT%H%M%S") + except ValueError: + dt = None + return { + "raw": raw, + "timezone": timezone or None, + "date_only": date_only, + "iso": dt.isoformat() if dt else None, + } + + +def _address(value: str) -> str: + v = value.strip() + if v.lower().startswith("mailto:"): + return v[7:] + return v + + +def _urls(values: Iterable[str]) -> list[str]: + out: list[str] = [] + seen: set[str] = set() + for value in values: + for match in _URL_RE.findall(value or ""): + url = match.rstrip(".,);]>") + if url not in seen: + seen.add(url) + out.append(url) + return out + + +def _video_urls(urls: Iterable[str]) -> list[str]: + result: list[str] = [] + for url in urls: + try: + host = (urlparse(url).hostname or "").lower() + except ValueError: + continue + if host in _VIDEO_HOSTS or any(host.endswith("." + item) for item in _VIDEO_HOSTS): + result.append(url) + continue + if any(token in host for token in ("zoom", "meet", "teams", "webex", "whereby", "jitsi")): + result.append(url) + return result + + +def parse_ics(text: str) -> dict[str, Any]: + """Parse VEVENT fields locally without resolving URLs or contacting organizers.""" + lines = _unfold_ical(text) + method = "" + events: list[dict[str, Any]] = [] + current: dict[str, Any] | None = None + raw_props: list[tuple[str, dict[str, str], str]] = [] + + for line in lines: + parsed = _split_property(line) + if not parsed: + continue + name, params, value = parsed + if name == "METHOD" and current is None: + method = value.upper() + continue + if name == "BEGIN" and value.upper() == "VEVENT": + current = { + "attendees": [], + "urls": [], + "video_urls": [], + "x_properties": {}, + } + raw_props = [] + continue + if name == "END" and value.upper() == "VEVENT" and current is not None: + text_values = [ + str(current.get("url") or ""), + str(current.get("location") or ""), + str(current.get("description") or ""), + ] + for key, val in current.get("x_properties", {}).items(): + if any(token in key for token in ("CONFERENCE", "MEETING", "HANGOUT", "ONLINE")): + text_values.append(str(val)) + urls = _urls(text_values) + current["urls"] = urls + current["video_urls"] = _video_urls(urls) + events.append(current) + current = None + raw_props = [] + continue + if current is None: + continue + raw_props.append(parsed) + clean = _ical_unescape(value) + if name == "UID": + current["uid"] = clean + elif name == "SUMMARY": + current["summary"] = clean + elif name == "DESCRIPTION": + current["description"] = clean + elif name == "LOCATION": + current["location"] = clean + elif name == "URL": + current["url"] = clean + elif name == "DTSTART": + current["start"] = _parse_dt(value, params) + elif name == "DTEND": + current["end"] = _parse_dt(value, params) + elif name == "DTSTAMP": + current["dtstamp"] = _parse_dt(value, params) + elif name == "SEQUENCE": + try: + current["sequence"] = int(value) + except ValueError: + current["sequence"] = value + elif name == "STATUS": + current["status"] = value.upper() + elif name == "ORGANIZER": + current["organizer"] = { + "address": _address(clean), + "name": params.get("CN") or None, + } + elif name == "ATTENDEE": + current["attendees"].append( + { + "address": _address(clean), + "name": params.get("CN") or None, + "role": params.get("ROLE") or None, + "partstat": params.get("PARTSTAT") or None, + "rsvp": params.get("RSVP") or None, + } + ) + elif name.startswith("X-"): + current["x_properties"][name] = clean + + invitation = bool(events) and method in {"REQUEST", "PUBLISH", "ADD", "CANCEL", "REPLY", "COUNTER", "DECLINECOUNTER"} + return { + "ok": True, + "method": method or None, + "is_calendar": bool(events), + "is_invitation": invitation, + "events": events, + "network_access": False, + "external_actions": False, + } + + +def _part_filename(part: Message) -> str: + try: + return str(part.get_filename() or "") + except Exception: + return "" + + +def detect_calendar_parts(message_or_bytes: Message | bytes) -> dict[str, Any]: + """Inspect headers and MIME parts. This function is deliberately zero-network/read-only.""" + message = ( + BytesParser(policy=default).parsebytes(message_or_bytes) + if isinstance(message_or_bytes, (bytes, bytearray)) + else message_or_bytes + ) + hints: list[str] = [] + headers = {str(k).lower(): str(v) for k, v in message.items()} + for name in ("content-class", "x-microsoft-cdo-message-class", "x-ms-has-attach"): + value = headers.get(name, "") + if "calend" in value.lower() or "appointment" in value.lower(): + hints.append(f"header:{name}") + + parts: list[dict[str, Any]] = [] + walk = list(message.walk()) if message.is_multipart() else [message] + for index, part in enumerate(walk): + content_type = str(part.get_content_type() or "").lower() + filename = _part_filename(part) + calendar_like = content_type in _CALENDAR_TYPES or filename.lower().endswith((".ics", ".ifb", ".vcs")) + if not calendar_like: + continue + try: + payload = part.get_payload(decode=True) + except Exception: + payload = None + if payload is None: + try: + content = part.get_content() + text = content if isinstance(content, str) else str(content) + except Exception: + text = "" + else: + charset = part.get_content_charset() or "utf-8" + try: + text = payload.decode(charset, errors="replace") + except LookupError: + text = payload.decode("utf-8", errors="replace") + parsed = parse_ics(text) if text else {"ok": True, "is_calendar": True, "is_invitation": False, "events": []} + parts.append( + { + "part_index": index, + "content_type": content_type, + "content_disposition": str(part.get_content_disposition() or ""), + "filename": filename or None, + "method_parameter": str(part.get_param("method") or "").upper() or None, + "sha256": hashlib.sha256((payload or text.encode("utf-8", "replace"))).hexdigest(), + "calendar": parsed, + } + ) + return { + "ok": True, + "calendar_candidate": bool(parts or hints), + "calendar_parts": parts, + "header_hints": hints, + "is_invitation": any(bool(p.get("calendar", {}).get("is_invitation")) for p in parts), + "network_access": False, + "external_actions": False, + } + + +def _fold_ical_line(line: str, limit: int = 75) -> list[str]: + # RFC 5545 fold limit is octets. Keep ASCII output where possible; UTF-8-safe fallback. + encoded = line.encode("utf-8") + if len(encoded) <= limit: + return [line] + out: list[str] = [] + current = "" + current_len = 0 + for ch in line: + size = len(ch.encode("utf-8")) + if current and current_len + size > (limit if not out else limit - 1): + out.append(current) + current = " " + ch + current_len = 1 + size + else: + current += ch + current_len += size + if current: + out.append(current) + return out + + +def build_calendar_request( + *, + uid: str, + summary: str, + start_utc: datetime, + end_utc: datetime, + organizer_email: str, + attendees: Iterable[str], + description: str = "", + location: str = "", + video_url: str | None = None, + sequence: int = 0, + prodid: str = "-//Postmaster MCP//v9.9//EN", +) -> str: + if start_utc.tzinfo is None or end_utc.tzinfo is None: + raise CalendarError("Calendar invitation start/end must be timezone-aware") + start = start_utc.astimezone(UTC) + end = end_utc.astimezone(UTC) + if end <= start: + raise CalendarError("Calendar invitation end must be after start") + attendee_list = [str(x).strip() for x in attendees if str(x).strip()] + if not attendee_list: + raise CalendarError("Calendar invitation requires at least one attendee") + if not organizer_email.strip(): + raise CalendarError("Calendar invitation requires organizer email") + + def stamp(dt: datetime) -> str: + return dt.astimezone(UTC).strftime("%Y%m%dT%H%M%SZ") + + desc = description + if video_url and video_url not in desc: + desc = (desc + "\n\n" if desc else "") + f"Video call: {video_url}" + lines = [ + "BEGIN:VCALENDAR", + "VERSION:2.0", + f"PRODID:{_ical_escape(prodid)}", + "CALSCALE:GREGORIAN", + "METHOD:REQUEST", + "BEGIN:VEVENT", + f"UID:{_ical_escape(uid)}", + f"DTSTAMP:{stamp(datetime.now(UTC))}", + f"DTSTART:{stamp(start)}", + f"DTEND:{stamp(end)}", + f"SEQUENCE:{int(sequence)}", + "STATUS:CONFIRMED", + f"SUMMARY:{_ical_escape(summary)}", + f"ORGANIZER:mailto:{organizer_email.strip()}", + ] + for attendee in attendee_list: + lines.append(f"ATTENDEE;ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:{attendee}") + if location: + lines.append(f"LOCATION:{_ical_escape(location)}") + if desc: + lines.append(f"DESCRIPTION:{_ical_escape(desc)}") + if video_url: + lines.append(f"URL:{video_url}") + lines.append(f"X-POSTMASTER-VIDEOCALL:{video_url}") + lines.extend(["END:VEVENT", "END:VCALENDAR"]) + folded: list[str] = [] + for line in lines: + folded.extend(_fold_ical_line(line)) + return "\r\n".join(folded) + "\r\n" + + +def job_calendar_window(job: dict[str, Any], *, default_duration_minutes: int = 30) -> tuple[datetime, datetime]: + """Resolve explicit calendar payload first, then one-time ISO schedule metadata.""" + payload = dict(job.get("payload") or {}) + cal = dict(payload.get("calendar") or {}) + start_raw = cal.get("start") or payload.get("calendar_start") + end_raw = cal.get("end") or payload.get("calendar_end") + if not start_raw and str(job.get("schedule_type") or "").lower() in {"once", "at", "datetime"}: + start_raw = job.get("schedule_value") + if not start_raw: + raise CalendarError("Task does not contain a concrete calendar start time") + + def parse(value: Any) -> datetime: + text = str(value or "").strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + dt = datetime.fromisoformat(text) + if dt.tzinfo is None: + raise CalendarError("Calendar task times must include a timezone offset") + return dt + + start = parse(start_raw) + end = parse(end_raw) if end_raw else start + timedelta(minutes=int(cal.get("duration_minutes") or default_duration_minutes)) + if end <= start: + raise CalendarError("Calendar task end must be after start") + return start, end diff --git a/src/postmaster/email_search_v990.py b/src/postmaster/email_search_v990.py new file mode 100644 index 00000000..98698d45 --- /dev/null +++ b/src/postmaster/email_search_v990.py @@ -0,0 +1,445 @@ +from __future__ import annotations + +from contextlib import closing + +import hashlib +import io +import json +import math +import re +import sqlite3 +import threading +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from email.message import Message +from email.parser import BytesParser +from email.policy import default +from html import unescape +from pathlib import Path +from typing import Any, Callable, Iterable, Mapping, Sequence + +import numpy as np + +try: + from pypdf import PdfReader +except Exception: # pragma: no cover - optional at import time in unit environments + PdfReader = None # type: ignore[assignment] + + +class EmailSearchError(ValueError): + pass + + +_TEXT_TYPES = { + "text/plain", + "text/csv", + "text/markdown", + "text/x-markdown", + "application/json", + "application/xml", + "text/xml", + "application/yaml", + "application/x-yaml", +} +_TOKEN_RE = re.compile(r"[\w@.+-]{2,}", re.UNICODE) +_TAG_RE = re.compile(r"<[^>]+>") + + +def _now_iso() -> str: + return datetime.now(UTC).isoformat().replace("+00:00", "Z") + + +def _clean_html(text: str) -> str: + text = re.sub(r"(?is)<(script|style)\b.*?>.*?", " ", text) + text = re.sub(r"(?i)|||", "\n", text) + text = _TAG_RE.sub(" ", text) + text = unescape(text) + return re.sub(r"[ \t]+", " ", re.sub(r"\n{3,}", "\n\n", text)).strip() + + +def _decode_part(part: Message, payload: bytes) -> str: + charset = part.get_content_charset() or "utf-8" + try: + return payload.decode(charset, errors="replace") + except LookupError: + return payload.decode("utf-8", errors="replace") + + +def extract_attachment_text(message_or_bytes: Message | bytes, *, max_total_chars: int = 200_000, max_part_chars: int = 80_000) -> list[dict[str, Any]]: + """Extract local text from supported MIME attachments; never resolve external URLs.""" + message = BytesParser(policy=default).parsebytes(message_or_bytes) if isinstance(message_or_bytes, (bytes, bytearray)) else message_or_bytes + results: list[dict[str, Any]] = [] + remaining = max(0, int(max_total_chars)) + for index, part in enumerate(message.walk() if message.is_multipart() else [message]): + filename = str(part.get_filename() or "") + disposition = str(part.get_content_disposition() or "") + if not filename and disposition != "attachment": + continue + ctype = str(part.get_content_type() or "").lower() + payload = part.get_payload(decode=True) or b"" + if not payload or remaining <= 0: + continue + text = "" + error = None + try: + if ctype in _TEXT_TYPES or filename.lower().endswith((".txt", ".md", ".csv", ".json", ".xml", ".yaml", ".yml")): + text = _decode_part(part, payload) + elif ctype == "text/html" or filename.lower().endswith((".html", ".htm")): + text = _clean_html(_decode_part(part, payload)) + elif ctype == "application/pdf" or filename.lower().endswith(".pdf"): + if PdfReader is None: + error = "pypdf unavailable" + else: + reader = PdfReader(io.BytesIO(payload)) + chunks: list[str] = [] + used = 0 + for page in reader.pages: + value = page.extract_text() or "" + chunks.append(value) + used += len(value) + if used >= max_part_chars: + break + text = "\n".join(chunks) + except Exception as exc: + error = f"{type(exc).__name__}: {exc}" + if text: + text = text[: min(max_part_chars, remaining)] + remaining -= len(text) + if text or error: + results.append( + { + "part_index": index, + "filename": filename or None, + "content_type": ctype, + "text": text, + "chars": len(text), + "error": error, + "sha256": hashlib.sha256(payload).hexdigest(), + "network_access": False, + } + ) + return results + + +def _vector_blob(vector: Sequence[float]) -> bytes: + arr = np.asarray(vector, dtype=np.float32).reshape(-1) + if arr.size == 0: + raise EmailSearchError("Semantic embedding is empty") + norm = float(np.linalg.norm(arr)) + if norm > 0: + arr = arr / norm + return arr.tobytes(order="C") + + +def _blob_vector(blob: bytes, dimensions: int) -> np.ndarray: + arr = np.frombuffer(blob, dtype=np.float32) + if arr.size != dimensions: + raise EmailSearchError("Stored email embedding dimension mismatch") + return arr + + +def _fts_query(query: str) -> str: + tokens = [t for t in _TOKEN_RE.findall(query or "") if t.strip(".-+")] + if not tokens: + return "" + # Quote every token to avoid exposing FTS syntax through MCP input. + return " AND ".join('"' + token.replace('"', '""') + '"' for token in tokens[:32]) + + +def _search_text(doc: Mapping[str, Any]) -> str: + attachments = doc.get("attachments_text") or "" + return "\n".join( + str(doc.get(key) or "") + for key in ("from_address", "to_address", "cc", "subject", "body_text", "snippet") + ) + ("\n" + str(attachments) if attachments else "") + + +class HybridEmailIndex: + """Persistent project-local email index: SQLite FTS5 + optional Model2Vec-compatible embeddings.""" + + def __init__( + self, + db_path: str | Path = "/data/email-search-v990.db", + *, + embed_one: Callable[[str], Sequence[float]] | None = None, + model_id: str = "postmaster-context-model", + ): + self.db_path = str(db_path) + self.embed_one = embed_one + self.model_id = str(model_id) + self._lock = threading.RLock() + Path(self.db_path).parent.mkdir(parents=True, exist_ok=True) + self._init_db() + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path, timeout=20.0) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA foreign_keys=ON") + return conn + + def _init_db(self) -> None: + with closing(self._connect()) as conn: + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS email_search_docs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + account_id TEXT NOT NULL, + mailbox TEXT NOT NULL, + uid TEXT NOT NULL, + message_id TEXT, + date_utc TEXT, + from_address TEXT, + to_address TEXT, + cc TEXT, + subject TEXT, + snippet TEXT, + body_text TEXT, + attachments_text TEXT, + attachments_json TEXT NOT NULL DEFAULT '[]', + content_hash TEXT NOT NULL, + indexed_at TEXT NOT NULL, + UNIQUE(account_id, mailbox, uid) + ); + CREATE VIRTUAL TABLE IF NOT EXISTS email_search_fts USING fts5( + from_address, to_address, cc, subject, snippet, body_text, attachments_text, + content='email_search_docs', content_rowid='id', tokenize='unicode61 remove_diacritics 2' + ); + CREATE TABLE IF NOT EXISTS email_search_embeddings ( + doc_id INTEGER PRIMARY KEY REFERENCES email_search_docs(id) ON DELETE CASCADE, + model_id TEXT NOT NULL, + dimensions INTEGER NOT NULL, + vector BLOB NOT NULL, + content_hash TEXT NOT NULL, + indexed_at TEXT NOT NULL + ); + CREATE TRIGGER IF NOT EXISTS email_search_docs_ai AFTER INSERT ON email_search_docs BEGIN + INSERT INTO email_search_fts(rowid,from_address,to_address,cc,subject,snippet,body_text,attachments_text) + VALUES(new.id,new.from_address,new.to_address,new.cc,new.subject,new.snippet,new.body_text,new.attachments_text); + END; + CREATE TRIGGER IF NOT EXISTS email_search_docs_ad AFTER DELETE ON email_search_docs BEGIN + INSERT INTO email_search_fts(email_search_fts,rowid,from_address,to_address,cc,subject,snippet,body_text,attachments_text) + VALUES('delete',old.id,old.from_address,old.to_address,old.cc,old.subject,old.snippet,old.body_text,old.attachments_text); + END; + CREATE TRIGGER IF NOT EXISTS email_search_docs_au AFTER UPDATE ON email_search_docs BEGIN + INSERT INTO email_search_fts(email_search_fts,rowid,from_address,to_address,cc,subject,snippet,body_text,attachments_text) + VALUES('delete',old.id,old.from_address,old.to_address,old.cc,old.subject,old.snippet,old.body_text,old.attachments_text); + INSERT INTO email_search_fts(rowid,from_address,to_address,cc,subject,snippet,body_text,attachments_text) + VALUES(new.id,new.from_address,new.to_address,new.cc,new.subject,new.snippet,new.body_text,new.attachments_text); + END; + CREATE INDEX IF NOT EXISTS idx_email_search_scope ON email_search_docs(account_id, mailbox, date_utc); + """ + ) + + def upsert(self, doc: Mapping[str, Any], *, embed: bool = True) -> dict[str, Any]: + account_id = str(doc.get("account_id") or "").strip() + mailbox = str(doc.get("mailbox") or "INBOX").strip() + uid = str(doc.get("uid") or "").strip() + if not account_id or not mailbox or not uid: + raise EmailSearchError("Email index requires account_id, mailbox and uid") + normalized = { + "message_id": str(doc.get("message_id") or ""), + "date_utc": str(doc.get("date_utc") or doc.get("date") or ""), + "from_address": str(doc.get("from_address") or doc.get("from") or ""), + "to_address": str(doc.get("to_address") or doc.get("to") or ""), + "cc": str(doc.get("cc") or ""), + "subject": str(doc.get("subject") or ""), + "snippet": str(doc.get("snippet") or ""), + "body_text": str(doc.get("body_text") or doc.get("body") or ""), + "attachments_text": str(doc.get("attachments_text") or ""), + } + attachment_meta = doc.get("attachments") or [] + content = _search_text(normalized) + content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest() + now = _now_iso() + with self._lock, closing(self._connect()) as conn: + existing = conn.execute( + "SELECT id,content_hash FROM email_search_docs WHERE account_id=? AND mailbox=? AND uid=?", + (account_id, mailbox, uid), + ).fetchone() + if existing and str(existing["content_hash"]) == content_hash: + doc_id = int(existing["id"]) + changed = False + else: + conn.execute( + """ + INSERT INTO email_search_docs( + account_id,mailbox,uid,message_id,date_utc,from_address,to_address,cc,subject,snippet,body_text, + attachments_text,attachments_json,content_hash,indexed_at + ) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + ON CONFLICT(account_id,mailbox,uid) DO UPDATE SET + message_id=excluded.message_id,date_utc=excluded.date_utc,from_address=excluded.from_address, + to_address=excluded.to_address,cc=excluded.cc,subject=excluded.subject,snippet=excluded.snippet, + body_text=excluded.body_text,attachments_text=excluded.attachments_text, + attachments_json=excluded.attachments_json,content_hash=excluded.content_hash,indexed_at=excluded.indexed_at + """, + ( + account_id, mailbox, uid, normalized["message_id"], normalized["date_utc"], normalized["from_address"], + normalized["to_address"], normalized["cc"], normalized["subject"], normalized["snippet"], + normalized["body_text"], normalized["attachments_text"], + json.dumps(attachment_meta, ensure_ascii=False, sort_keys=True), content_hash, now, + ), + ) + row = conn.execute( + "SELECT id FROM email_search_docs WHERE account_id=? AND mailbox=? AND uid=?", + (account_id, mailbox, uid), + ).fetchone() + doc_id = int(row["id"]) + conn.execute("DELETE FROM email_search_embeddings WHERE doc_id=?", (doc_id,)) + changed = True + semantic = False + if embed and self.embed_one is not None: + current = conn.execute( + "SELECT model_id,content_hash FROM email_search_embeddings WHERE doc_id=?", (doc_id,) + ).fetchone() + if not current or str(current["model_id"]) != self.model_id or str(current["content_hash"]) != content_hash: + vector = np.asarray(self.embed_one(content), dtype=np.float32).reshape(-1) + blob = _vector_blob(vector) + conn.execute( + "INSERT OR REPLACE INTO email_search_embeddings(doc_id,model_id,dimensions,vector,content_hash,indexed_at) VALUES(?,?,?,?,?,?)", + (doc_id, self.model_id, int(vector.size), blob, content_hash, now), + ) + semantic = True + conn.commit() + return {"ok": True, "doc_id": doc_id, "changed": changed, "semantic_indexed": semantic, "content_hash": content_hash} + + def search( + self, + query: str, + *, + account_id: str, + mailbox: str | None = None, + since_days: int | None = 90, + limit: int = 20, + semantic_weight: float = 0.60, + lexical_weight: float = 0.40, + semantic_scan_limit: int = 5000, + ) -> dict[str, Any]: + q = str(query or "").strip() + if not q: + raise EmailSearchError("Hybrid email search query is required") + limit = max(1, min(int(limit), 100)) + scope_sql = ["d.account_id=?"] + params: list[Any] = [str(account_id)] + if mailbox: + scope_sql.append("d.mailbox=?") + params.append(str(mailbox)) + if since_days is not None and int(since_days) >= 0: + cutoff = datetime.now(UTC) - timedelta(days=int(since_days)) + scope_sql.append("(d.date_utc='' OR d.date_utc IS NULL OR d.date_utc>=?)") + params.append(cutoff.isoformat()) + scope = " AND ".join(scope_sql) + lex: dict[int, tuple[int, float]] = {} + fts = _fts_query(q) + with closing(self._connect()) as conn: + if fts: + rows = conn.execute( + f""" + SELECT d.id, bm25(email_search_fts, 1.2,1.0,0.8,2.5,0.8,1.5,1.8) AS score + FROM email_search_fts JOIN email_search_docs d ON d.id=email_search_fts.rowid + WHERE email_search_fts MATCH ? AND {scope} + ORDER BY score ASC LIMIT ? + """, + [fts, *params, max(limit * 8, 100)], + ).fetchall() + for rank, row in enumerate(rows, 1): + lex[int(row["id"])] = (rank, float(row["score"])) + + sem: dict[int, tuple[int, float]] = {} + semantic_available = False + if self.embed_one is not None: + qv = np.asarray(self.embed_one(q), dtype=np.float32).reshape(-1) + qnorm = float(np.linalg.norm(qv)) + if qnorm > 0: + qv = qv / qnorm + rows = conn.execute( + f""" + SELECT d.id,e.dimensions,e.vector FROM email_search_docs d + JOIN email_search_embeddings e ON e.doc_id=d.id + WHERE e.model_id=? AND {scope} + ORDER BY d.date_utc DESC LIMIT ? + """, + [self.model_id, *params, max(1, min(int(semantic_scan_limit), 20000))], + ).fetchall() + scored: list[tuple[int, float]] = [] + for row in rows: + if int(row["dimensions"]) != int(qv.size): + continue + dv = _blob_vector(row["vector"], int(row["dimensions"])) + scored.append((int(row["id"]), float(np.dot(qv, dv)))) + scored.sort(key=lambda item: item[1], reverse=True) + sem = {doc_id: (rank, score) for rank, (doc_id, score) in enumerate(scored, 1)} + semantic_available = bool(scored) + + ids = set(lex) | set(sem) + fused: list[tuple[int, float]] = [] + k = 60.0 + for doc_id in ids: + score = 0.0 + if doc_id in lex: + score += float(lexical_weight) / (k + lex[doc_id][0]) + if doc_id in sem: + score += float(semantic_weight) / (k + sem[doc_id][0]) + fused.append((doc_id, score)) + fused.sort(key=lambda item: item[1], reverse=True) + selected = fused[:limit] + if not selected: + return { + "ok": True, "query": q, "results": [], "count": 0, + "semantic_active": semantic_available, "lexical_active": bool(fts), + "index_only": True, + } + placeholders = ",".join("?" for _ in selected) + docs = conn.execute( + f"SELECT * FROM email_search_docs WHERE id IN ({placeholders})", [doc_id for doc_id, _ in selected] + ).fetchall() + by_id = {int(row["id"]): row for row in docs} + results: list[dict[str, Any]] = [] + for doc_id, fused_score in selected: + row = by_id.get(doc_id) + if row is None: + continue + results.append( + { + "account_id": row["account_id"], "mailbox": row["mailbox"], "uid": row["uid"], + "message_id": row["message_id"], "date_utc": row["date_utc"], "from": row["from_address"], + "to": row["to_address"], "cc": row["cc"], "subject": row["subject"], "snippet": row["snippet"], + "body_indexed": bool(row["body_text"]), "attachment_text_indexed": bool(row["attachments_text"]), + "attachments": json.loads(row["attachments_json"] or "[]"), "score": fused_score, + "lexical_rank": lex.get(doc_id, (None, None))[0], "semantic_rank": sem.get(doc_id, (None, None))[0], + "semantic_similarity": sem.get(doc_id, (None, None))[1], + } + ) + return { + "ok": True, "query": q, "results": results, "count": len(results), + "semantic_active": semantic_available, "lexical_active": bool(fts), "index_only": True, + } + + def status(self) -> dict[str, Any]: + with closing(self._connect()) as conn: + docs = int(conn.execute("SELECT COUNT(*) FROM email_search_docs").fetchone()[0]) + vectors = int(conn.execute("SELECT COUNT(*) FROM email_search_embeddings WHERE model_id=?", (self.model_id,)).fetchone()[0]) + attachment_docs = int(conn.execute("SELECT COUNT(*) FROM email_search_docs WHERE attachments_text<>''").fetchone()[0]) + return { + "ok": True, "db_path": self.db_path, "documents": docs, "semantic_documents": vectors, + "attachment_text_documents": attachment_docs, "fts5": True, "model_id": self.model_id, + "external_resource_fetch": False, + } + + +def combine_attachment_text(extracted: Iterable[Mapping[str, Any]]) -> tuple[str, list[dict[str, Any]]]: + chunks: list[str] = [] + metadata: list[dict[str, Any]] = [] + for row in extracted: + text = str(row.get("text") or "") + filename = str(row.get("filename") or "") + if text: + chunks.append(f"[{filename or 'attachment'}]\n{text}") + metadata.append({ + "filename": filename or None, + "content_type": row.get("content_type"), + "chars": int(row.get("chars") or len(text)), + "sha256": row.get("sha256"), + "extraction_error": row.get("error"), + }) + return "\n\n".join(chunks), metadata diff --git a/src/postmaster/runtime.py b/src/postmaster/runtime.py index 334d0f25..8872524c 100644 --- a/src/postmaster/runtime.py +++ b/src/postmaster/runtime.py @@ -25,6 +25,8 @@ from .runtime_v968 import install_runtime_v968 from .runtime_v969 import install_runtime_v969_mcp, install_runtime_v969_pre_webgui from .runtime_v980 import install_runtime_v980 +from .runtime_v990 import install_runtime_v990 +from .whatsapp_runtime_v990 import install_whatsapp_runtime_v990 from .tracking_telemetry_v971 import ( install_tracking_telemetry_v971, install_tracking_webgui_v971, @@ -39,6 +41,7 @@ ) from .webgui_release_identity import install_webgui_release_identity, project_release_version from .webgui_structured_data_v980 import install_webgui_structured_data_v980 +from .webgui_whatsapp_v990 import install_webgui_whatsapp_v990 from .webgui_tasks import task_fragment as _task_fragment_v945 from .webgui_v945 import install_webgui_v945 from .webgui_v951 import install_webgui_v951 @@ -54,6 +57,7 @@ from .webgui_v963_high_noise import install_webgui_v963_high_noise from .webgui_v964 import install_webgui_v964 from .webgui_v970 import install_webgui_v970 +from . import webgui_v970 as _webgui_v970 from .webgui_visual_restoration import install_webgui_visual_restoration for _name in dir(_core): @@ -161,6 +165,14 @@ def _task_dashboard_fragment(request): # unchanged. Destructive DDL is review-only and row deletes require explicit confirmation. runtime_status = install_runtime_v980(_base, _core, runtime_status) install_webgui_structured_data_v980(app, _base) +# v9.9.0 source overlays: email/calendar policy + hybrid search first, then the +# fail-closed clean-room WhatsApp surface. Real WhatsApp acceptance remains false +# until controlled protocol/multi-device interoperability has been demonstrated. +_v990 = install_runtime_v990(_base, _core, runtime_status) +runtime_status = _v990["runtime_status"] +_wa_v990 = install_whatsapp_runtime_v990(_base, _core, runtime_status) +runtime_status = _wa_v990["runtime_status"] +install_webgui_whatsapp_v990(app, _base, _webgui_v962, _webgui_v962_views, _webgui_v970) # The lazy shell originates in v9.6.2 but must identify the release that is actually loaded. # VERSION is local release metadata, so this does not add a network lookup to WebGUI rendering. install_webgui_release_identity(_webgui_v962, project_release_version()) diff --git a/src/postmaster/runtime_v990.py b/src/postmaster/runtime_v990.py new file mode 100644 index 00000000..aaffe19a --- /dev/null +++ b/src/postmaster/runtime_v990.py @@ -0,0 +1,774 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +from datetime import UTC, datetime +from typing import Any, Callable, Mapping, Sequence + +from mcp.types import ToolAnnotations + +from .calendar_v990 import build_calendar_request, detect_calendar_parts, job_calendar_window +from .email_search_v990 import EmailSearchError, HybridEmailIndex, combine_attachment_text +from .tracking_policy_v990 import ( + TrackingApprovalError, + TrackingApprovalStore, + attachment_intent_digest, + body_digest, + classify_open_event, + effective_tracking, +) + +MCP_COMMAND_COUNT_V980 = 118 +MCP_EMAIL_CALENDAR_COMMANDS_V990 = 4 +# WhatsApp tools are installed by whatsapp_runtime_v990 and counted here for one release identity. +MCP_WHATSAPP_COMMANDS_V990 = 8 +MCP_COMMAND_COUNT_V990 = MCP_COMMAND_COUNT_V980 + MCP_EMAIL_CALENDAR_COMMANDS_V990 + MCP_WHATSAPP_COMMANDS_V990 + + +def _annotations(*, read_only: bool, destructive: bool = False, idempotent: bool = False): + return ToolAnnotations( + read_only_hint=read_only, + destructive_hint=destructive, + idempotent_hint=idempotent, + open_world_hint=False, + ) + + +def _as_dict(value: Any) -> dict[str, Any]: + return dict(value) if isinstance(value, Mapping) else {} + + +def _safe(fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: + try: + return fn(*args, **kwargs) + except (TrackingApprovalError, EmailSearchError, ValueError) as exc: + return {"ok": False, "error": str(exc)} + except Exception as exc: # preserve the project's MCP no-crash convention + return {"ok": False, "error": f"{type(exc).__name__}: {exc}"} + + +def _account(base: Any, account_id: str | None) -> dict[str, Any]: + store = base.account_store() + row = store.get_account(account_id) + if not isinstance(row, Mapping): + raise ValueError("Unable to resolve selected email account") + return dict(row) + + +def _account_id(base: Any, account_id: str | None) -> str: + row = _account(base, account_id) + resolved = str(row.get("id") or row.get("account_id") or "").strip() + if not resolved: + raise ValueError("Selected email account has no account id") + return resolved + + +def _tracking_default(base: Any, account_id: str | None) -> bool: + row = _account(base, account_id) + return bool(row.get("tracking_default", False)) + + +def _normalize_recipients(value: Sequence[str] | None) -> list[str]: + return [str(x).strip() for x in (value or []) if str(x).strip()] + + +def _send_intent( + operation: str, + *, + account_id: str, + to: Sequence[str] | None = None, + mailbox: str | None = None, + uid: str | None = None, + subject: str | None = None, + body: str = "", + body_html: str | None = None, + body_amp: str | None = None, + cc: Sequence[str] | None = None, + bcc: Sequence[str] | None = None, + attachments: Any = None, + campaign_id: str | None = None, + idempotency_key: str | None = None, + extras: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + return { + "operation": operation, + "account_id": account_id, + "to": _normalize_recipients(to), + "mailbox": str(mailbox or ""), + "uid": str(uid or ""), + "subject": str(subject or ""), + "body_sha256": body_digest(body), + "body_html_sha256": body_digest(body_html), + "body_amp_sha256": body_digest(body_amp), + "cc": _normalize_recipients(cc), + "bcc": _normalize_recipients(bcc), + "attachments_sha256": attachment_intent_digest(attachments), + "campaign_id": str(campaign_id or ""), + "idempotency_key": str(idempotency_key or ""), + "extras": dict(extras or {}), + } + + +def _approval_summary(intent: Mapping[str, Any]) -> dict[str, Any]: + return { + "operation": intent.get("operation"), + "account_id": intent.get("account_id"), + "to": intent.get("to"), + "mailbox": intent.get("mailbox"), + "uid": intent.get("uid"), + "subject": intent.get("subject"), + "tracking": "off", + } + + +def _guard_tracking_off( + store: TrackingApprovalStore, + *, + operation: str, + intent: Mapping[str, Any], + effective: bool, + approval_id: str | None, +) -> dict[str, Any] | None: + if effective: + return None + if not approval_id: + return store.create(operation=operation, intent=intent, summary=_approval_summary(intent)) + try: + store.consume(approval_id, operation=operation, intent=intent) + except TrackingApprovalError as exc: + return { + "ok": False, + "approval_required": True, + "approval_type": "tracking_disabled", + "send_performed": False, + "error": str(exc), + "message": "Ask for fresh explicit chat approval before retrying this tracking-OFF action.", + } + return None + + +def _semantic_embedder(base: Any) -> tuple[Callable[[str], Sequence[float]] | None, str]: + """Adapt the existing local semantic runtime without downloading a second model.""" + try: + engine = base.context_engine() + except Exception: + return None, "lexical-only" + candidates = [engine, getattr(engine, "semantic", None), getattr(engine, "semantic_engine", None), getattr(getattr(engine, "store", None), "semantic", None)] + for obj in [x for x in candidates if x is not None]: + model_id = str(getattr(obj, "model_id", None) or getattr(obj, "model_name", None) or "postmaster-context-model") + for name in ("embed_one", "encode_one"): + fn = getattr(obj, name, None) + if callable(fn): + return lambda text, fn=fn: fn(text), model_id + fn = getattr(obj, "encode", None) + if callable(fn): + def encode_one(text: str, fn=fn): + result = fn([text]) + try: + return result[0] + except Exception: + return result + return encode_one, model_id + model = getattr(obj, "model", None) + fn = getattr(model, "encode", None) + if callable(fn): + return lambda text, fn=fn: fn([text])[0], model_id + return None, "lexical-only" + + +def _message_rows(result: Any) -> list[dict[str, Any]]: + if isinstance(result, list): + return [dict(x) for x in result if isinstance(x, Mapping)] + if isinstance(result, Mapping): + for key in ("emails", "messages", "results", "items"): + value = result.get(key) + if isinstance(value, list): + return [dict(x) for x in value if isinstance(x, Mapping)] + return [] + + +def _full_message(result: Any) -> dict[str, Any]: + if isinstance(result, Mapping): + if isinstance(result.get("email"), Mapping): + return dict(result["email"]) + return dict(result) + return {} + + +def _attachment_text_from_tools(core: Any, *, mailbox: str, uid: str, account_id: str, max_attachments: int = 12) -> tuple[str, list[dict[str, Any]]]: + try: + listing = core.list_email_attachments(mailbox=mailbox, uid=uid, account_id=account_id) + except TypeError: + listing = core.list_email_attachments(mailbox, uid, account_id) + rows = [] + if isinstance(listing, Mapping): + rows = listing.get("attachments") or listing.get("items") or [] + elif isinstance(listing, list): + rows = listing + extracted: list[dict[str, Any]] = [] + for idx, item in enumerate(rows[:max_attachments]): + if not isinstance(item, Mapping): + continue + filename = str(item.get("filename") or item.get("name") or "") + content_type = str(item.get("content_type") or item.get("media_type") or "") + try: + read = core.read_email_attachment( + mailbox=mailbox, + uid=uid, + filename=filename or None, + index=item.get("index", idx), + max_chars=80000, + account_id=account_id, + ) + except Exception as exc: + extracted.append({"filename": filename or None, "content_type": content_type, "text": "", "error": f"{type(exc).__name__}: {exc}"}) + continue + if isinstance(read, Mapping): + text = str(read.get("text") or read.get("content") or "") + extracted.append({ + "filename": filename or read.get("filename"), + "content_type": content_type or read.get("content_type"), + "text": text, + "chars": len(text), + "error": read.get("error"), + "network_access": False, + }) + return combine_attachment_text(extracted) + + +def _enrich_email_index( + *, + base: Any, + core: Any, + index: HybridEmailIndex, + query: str, + account_id: str, + mailbox: str, + since_days: int, + enrich_limit: int, +) -> dict[str, Any]: + # Do not rely only on lexical IMAP TEXT matching: semantic queries may use different words. + try: + discovery = core.search_emails( + mailbox=mailbox, + since_days=since_days, + unread_only=False, + limit=max(1, min(int(enrich_limit), 100)), + account_id=account_id, + include_timings=False, + ) + except TypeError: + discovery = core.search_emails(mailbox=mailbox, since_days=since_days, limit=max(1, min(int(enrich_limit), 100)), account_id=account_id) + rows = _message_rows(discovery) + indexed = 0 + failures: list[dict[str, str]] = [] + for row in rows: + uid = str(row.get("uid") or "").strip() + if not uid: + continue + try: + detail = _full_message(core.get_email(mailbox=mailbox, uid=uid, account_id=account_id)) + attachments_text, attachment_meta = _attachment_text_from_tools(core, mailbox=mailbox, uid=uid, account_id=account_id) + doc = { + "account_id": account_id, + "mailbox": mailbox, + "uid": uid, + "message_id": detail.get("message_id") or row.get("message_id"), + "date_utc": detail.get("date_utc") or detail.get("date") or row.get("date_utc") or row.get("date"), + "from_address": detail.get("from_address") or detail.get("from") or row.get("from_address") or row.get("from"), + "to_address": detail.get("to_address") or detail.get("to") or row.get("to_address") or row.get("to"), + "cc": detail.get("cc") or row.get("cc"), + "subject": detail.get("subject") or row.get("subject"), + "snippet": detail.get("snippet") or row.get("snippet"), + "body_text": detail.get("body_text") or detail.get("body") or "", + "attachments_text": attachments_text, + "attachments": attachment_meta, + } + index.upsert(doc, embed=True) + indexed += 1 + except Exception as exc: + failures.append({"uid": uid, "error": f"{type(exc).__name__}: {exc}"}) + return {"indexed": indexed, "discovered": len(rows), "failures": failures[:20], "query": query, "network_scope": "imap_only"} + + +def _calendar_parts_from_detail(core: Any, *, mailbox: str, uid: str, account_id: str, get_email_fn: Callable[..., Any] | None = None) -> dict[str, Any]: + """Read calendar material locally from full-message/raw or ICS attachments. + + Existing Postmaster readers vary by historical release. Prefer raw MIME if exposed; otherwise + inspect calendar attachment bytes/text through the existing attachment tools. No remote URL is fetched. + """ + reader = get_email_fn or core.get_email + detail_raw = reader(mailbox=mailbox, uid=uid, account_id=account_id) + detail = _full_message(detail_raw) + raw = detail.get("raw_bytes") or detail.get("raw") or detail.get("raw_message") + if isinstance(raw, str): + try: + raw = base64.b64decode(raw, validate=True) + except Exception: + raw = raw.encode("utf-8", errors="replace") + if isinstance(raw, (bytes, bytearray)): + return detect_calendar_parts(bytes(raw)) + + found: list[dict[str, Any]] = [] + try: + listing = core.list_email_attachments(mailbox=mailbox, uid=uid, account_id=account_id) + except Exception: + listing = [] + rows = listing.get("attachments", []) if isinstance(listing, Mapping) else (listing if isinstance(listing, list) else []) + for idx, item in enumerate(rows): + if not isinstance(item, Mapping): + continue + filename = str(item.get("filename") or item.get("name") or "") + ctype = str(item.get("content_type") or item.get("media_type") or "").lower() + if ctype not in {"text/calendar", "application/ics", "application/calendar", "application/icalendar"} and not filename.lower().endswith((".ics", ".vcs", ".ifb")): + continue + try: + downloaded = core.get_email_attachment( + mailbox=mailbox, uid=uid, filename=filename or None, index=item.get("index", idx), include_base64=True, account_id=account_id + ) + if not isinstance(downloaded, Mapping): + continue + b64 = downloaded.get("content_base64") or downloaded.get("base64") or downloaded.get("data_base64") + if b64: + payload = base64.b64decode(str(b64)) + from email.message import EmailMessage + msg = EmailMessage() + msg["Subject"] = str(detail.get("subject") or "") + msg.set_content("calendar attachment") + msg.add_attachment(payload, maintype="text", subtype="calendar", filename=filename or "invite.ics") + parsed = detect_calendar_parts(msg.as_bytes()) + found.extend(parsed.get("calendar_parts", [])) + except Exception: + continue + return { + "ok": True, + "calendar_detected": bool(found), + "calendar_parts": found, + "network_access": False, + "external_actions": False, + "detection_scope": "raw-mime-or-local-calendar-attachments", + } + + +def install_runtime_v990( + base: Any, + core: Any, + previous_runtime_status: Callable[[], Any], + *, + tracking_approval_db: str | None = None, + email_search_db: str | None = None, +) -> dict[str, Any]: + """Install v9.9 email/calendar policy and search overlays; WhatsApp installs separately. + + Persistent v9.9 stores are initialized lazily so importing the composed runtime remains + side-effect free in validation/test environments that intentionally do not mount /data. + Production still uses the persistent /data paths on first explicit feature use. + """ + approval_store_obj: TrackingApprovalStore | None = None + email_index_obj: HybridEmailIndex | None = None + embed_one: Callable[[str], Sequence[float]] | None = None + model_id = "lexical-only" + semantic_resolved = False + + def get_approval_store() -> TrackingApprovalStore: + nonlocal approval_store_obj + if approval_store_obj is None: + approval_store_obj = TrackingApprovalStore(tracking_approval_db or "/data/tracking-approvals-v990.db") + return approval_store_obj + + def get_email_index() -> HybridEmailIndex: + nonlocal email_index_obj, embed_one, model_id, semantic_resolved + if email_index_obj is None: + if not semantic_resolved: + embed_one, model_id = _semantic_embedder(base) + semantic_resolved = True + email_index_obj = HybridEmailIndex( + email_search_db or "/data/email-search-v990.db", + embed_one=embed_one, + model_id=model_id, + ) + return email_index_obj + + old_send = core.send_email + old_reply = core.reply_email + old_follow = core.follow_up_email + old_list_open_events = core.list_open_events + old_get_email = core.get_email + old_list_tracking_deliveries = getattr(core, "list_tracking_deliveries", None) + + def send_email( + to: list[str], + subject: str, + body: str = "", + cc: list[str] | None = None, + bcc: list[str] | None = None, + body_html: str | None = None, + body_amp: str | None = None, + attachments: list[dict[str, Any]] | None = None, + track_opens: bool | None = None, + campaign_id: str | None = None, + account_id: str | None = None, + newsletter_mode: bool = False, + unsubscribe_url: str | None = None, + unsubscribe_email: str | None = None, + one_click_unsubscribe: bool = False, + automatic_unsubscribe: bool = True, + dsn_notify_success: bool = False, + idempotency_key: str | None = None, + force_send: bool = False, + confirm_suppressed_recipients: list[str] | None = None, + tracking_disable_approval_id: str | None = None, + ): + """WRITE ACTION. Send email through the existing outbound pipeline. + + Generated subjects should prefer ASCII text/punctuation when practical; never silently rewrite + Unicode explicitly supplied by the user. Prefer a Postmaster Stored File/signed storage link over + embedding an attachment unless the user requests an attachment or the recipient/workflow requires it. + + If the effective tracking state is OFF -- including account-default OFF when track_opens=None -- do + not send until the user gives fresh explicit chat approval for this exact send. First call returns a + one-use tracking_disable preview id; retry only after that approval with tracking_disable_approval_id. + Suppressed-recipient approval remains a separate exact per-send boundary. + """ + aid = _account_id(base, account_id) + eff = effective_tracking(track_opens, account_tracking_default=_tracking_default(base, account_id)) + extras = { + "newsletter_mode": newsletter_mode, + "unsubscribe_url": unsubscribe_url or "", + "unsubscribe_email": unsubscribe_email or "", + "one_click_unsubscribe": one_click_unsubscribe, + "automatic_unsubscribe": automatic_unsubscribe, + "dsn_notify_success": dsn_notify_success, + "force_send": force_send, + "confirm_suppressed_recipients": sorted(confirm_suppressed_recipients or []), + } + intent = _send_intent( + "send_email", account_id=aid, to=to, subject=subject, body=body, body_html=body_html, body_amp=body_amp, + cc=cc, bcc=bcc, attachments=attachments, campaign_id=campaign_id, idempotency_key=idempotency_key, extras=extras, + ) + blocked = _guard_tracking_off(get_approval_store(), operation="send_email", intent=intent, effective=eff, approval_id=tracking_disable_approval_id) + if blocked: + return blocked + return old_send( + to=to, subject=subject, body=body, cc=cc, bcc=bcc, body_html=body_html, body_amp=body_amp, + attachments=attachments, track_opens=track_opens, campaign_id=campaign_id, account_id=account_id, + newsletter_mode=newsletter_mode, unsubscribe_url=unsubscribe_url, unsubscribe_email=unsubscribe_email, + one_click_unsubscribe=one_click_unsubscribe, automatic_unsubscribe=automatic_unsubscribe, + dsn_notify_success=dsn_notify_success, idempotency_key=idempotency_key, force_send=force_send, + confirm_suppressed_recipients=confirm_suppressed_recipients, + ) + + def _threaded(operation: str, delegate: Callable[..., Any], *, mailbox: str, uid: str, body: str, cc: list[str] | None, + bcc: list[str] | None, body_html: str | None, attachments: list[dict[str, Any]] | None, + track_opens: bool | None, campaign_id: str | None, account_id: str | None, newsletter_mode: bool, + unsubscribe_url: str | None, unsubscribe_email: str | None, one_click_unsubscribe: bool, + dsn_notify_success: bool, idempotency_key: str | None, force_send: bool, + confirm_suppressed_recipients: list[str] | None, tracking_disable_approval_id: str | None): + aid = _account_id(base, account_id) + eff = effective_tracking(track_opens, account_tracking_default=_tracking_default(base, account_id)) + extras = { + "newsletter_mode": newsletter_mode, + "unsubscribe_url": unsubscribe_url or "", + "unsubscribe_email": unsubscribe_email or "", + "one_click_unsubscribe": one_click_unsubscribe, + "dsn_notify_success": dsn_notify_success, + "force_send": force_send, + "confirm_suppressed_recipients": sorted(confirm_suppressed_recipients or []), + } + intent = _send_intent( + operation, account_id=aid, mailbox=mailbox, uid=uid, body=body, body_html=body_html, cc=cc, bcc=bcc, + attachments=attachments, campaign_id=campaign_id, idempotency_key=idempotency_key, extras=extras, + ) + blocked = _guard_tracking_off(get_approval_store(), operation=operation, intent=intent, effective=eff, approval_id=tracking_disable_approval_id) + if blocked: + return blocked + return delegate( + mailbox=mailbox, uid=uid, body=body, cc=cc, bcc=bcc, body_html=body_html, attachments=attachments, + track_opens=track_opens, campaign_id=campaign_id, account_id=account_id, newsletter_mode=newsletter_mode, + unsubscribe_url=unsubscribe_url, unsubscribe_email=unsubscribe_email, one_click_unsubscribe=one_click_unsubscribe, + dsn_notify_success=dsn_notify_success, idempotency_key=idempotency_key, force_send=force_send, + confirm_suppressed_recipients=confirm_suppressed_recipients, + ) + + def reply_email( + mailbox: str, uid: str, body: str = "", cc: list[str] | None = None, bcc: list[str] | None = None, + body_html: str | None = None, attachments: list[dict[str, Any]] | None = None, track_opens: bool | None = None, + campaign_id: str | None = None, account_id: str | None = None, newsletter_mode: bool = False, + unsubscribe_url: str | None = None, unsubscribe_email: str | None = None, one_click_unsubscribe: bool = False, + dsn_notify_success: bool = False, idempotency_key: str | None = None, force_send: bool = False, + confirm_suppressed_recipients: list[str] | None = None, tracking_disable_approval_id: str | None = None, + ): + """WRITE ACTION. Reply in-thread through the existing outbound pipeline. + + Prefer Stored File/signed storage links over attachments unless an attachment is requested/required. + Any effective tracking-OFF reply requires fresh one-use explicit chat approval before transmission. + """ + return _threaded("reply_email", old_reply, mailbox=mailbox, uid=uid, body=body, cc=cc, bcc=bcc, + body_html=body_html, attachments=attachments, track_opens=track_opens, campaign_id=campaign_id, + account_id=account_id, newsletter_mode=newsletter_mode, unsubscribe_url=unsubscribe_url, + unsubscribe_email=unsubscribe_email, one_click_unsubscribe=one_click_unsubscribe, + dsn_notify_success=dsn_notify_success, idempotency_key=idempotency_key, force_send=force_send, + confirm_suppressed_recipients=confirm_suppressed_recipients, + tracking_disable_approval_id=tracking_disable_approval_id) + + def follow_up_email( + mailbox: str, uid: str, body: str = "", cc: list[str] | None = None, bcc: list[str] | None = None, + body_html: str | None = None, attachments: list[dict[str, Any]] | None = None, track_opens: bool | None = None, + campaign_id: str | None = None, account_id: str | None = None, newsletter_mode: bool = False, + unsubscribe_url: str | None = None, unsubscribe_email: str | None = None, one_click_unsubscribe: bool = False, + dsn_notify_success: bool = False, idempotency_key: str | None = None, force_send: bool = False, + confirm_suppressed_recipients: list[str] | None = None, tracking_disable_approval_id: str | None = None, + ): + """WRITE ACTION. Follow up on an outbound message through the existing pipeline. + + Prefer Stored File/signed storage links over attachments unless an attachment is requested/required. + Any effective tracking-OFF follow-up requires fresh one-use explicit chat approval before transmission. + """ + return _threaded("follow_up_email", old_follow, mailbox=mailbox, uid=uid, body=body, cc=cc, bcc=bcc, + body_html=body_html, attachments=attachments, track_opens=track_opens, campaign_id=campaign_id, + account_id=account_id, newsletter_mode=newsletter_mode, unsubscribe_url=unsubscribe_url, + unsubscribe_email=unsubscribe_email, one_click_unsubscribe=one_click_unsubscribe, + dsn_notify_success=dsn_notify_success, idempotency_key=idempotency_key, force_send=force_send, + confirm_suppressed_recipients=confirm_suppressed_recipients, + tracking_disable_approval_id=tracking_disable_approval_id) + + def _delivery_sent_times(*, delivery_id: str | None, campaign_id: str | None, recipient: str | None, account_id: str | None) -> dict[str, Any]: + if not callable(old_list_tracking_deliveries): + return {} + try: + deliveries = old_list_tracking_deliveries( + campaign_id=campaign_id, recipient=recipient, account_id=account_id, limit=1000 + ) + except TypeError: + try: + deliveries = old_list_tracking_deliveries(campaign_id=campaign_id, limit=1000) + except Exception: + return {} + except Exception: + return {} + candidates = deliveries if isinstance(deliveries, list) else (deliveries.get("deliveries", []) if isinstance(deliveries, Mapping) else []) + result: dict[str, Any] = {} + for item in candidates: + if not isinstance(item, Mapping): + continue + did = str(item.get("delivery_id") or item.get("id") or "").strip() + if delivery_id and did != str(delivery_id): + continue + if did: + result[did] = item.get("sent_at") or item.get("created_at") or item.get("delivered_at") + return result + + def list_open_events(delivery_id: str | None = None, campaign_id: str | None = None, recipient: str | None = None, + account_id: str | None = None, limit: int = 500): + """Read-only. List observed open-image events with conservative timing classification. + + Google Image Proxy events more than 60 seconds after sent_at are marked likely_human_via_proxy, not + guaranteed-human. If the open row lacks sent_at, Postmaster resolves it from the matching delivery. + Proxy-derived IP/location/browser/OS remain unreliable regardless of timing. + """ + result = old_list_open_events(delivery_id=delivery_id, campaign_id=campaign_id, recipient=recipient, account_id=account_id, limit=limit) + if not isinstance(result, (list, Mapping)): + return result + rows = result if isinstance(result, list) else result.get("events", result.get("open_events", [])) + if not isinstance(rows, list): + return result + sent_times = _delivery_sent_times(delivery_id=delivery_id, campaign_id=campaign_id, recipient=recipient, account_id=account_id) + for row in rows: + if not isinstance(row, dict): + continue + did = str(row.get("delivery_id") or "").strip() + sent_at = row.get("sent_at") or row.get("delivery_sent_at") or sent_times.get(did) + row["open_classification"] = classify_open_event( + sent_at=sent_at, opened_at=row.get("opened_at") or row.get("created_at"), + user_agent=str(row.get("user_agent") or ""), client_source=str(row.get("client_source") or ""), + ) + if isinstance(result, Mapping): + out = dict(result) + key = "events" if "events" in result else ("open_events" if "open_events" in result else None) + if key: + out[key] = rows + out["classification_policy"] = {"google_image_proxy_human_delay_seconds": 60, "human_is_probabilistic": True, "proxy_metadata_reliable": False} + return out + return rows + + def get_email(mailbox: str, uid: str, account_id: str | None = None, inspection: str | None = None, + content_mode: str = "safe", acknowledge_unsanitized_content_risk: bool = False): + """Read one email and always attach local MIME/header calendar-invitation analysis. + + Calendar detection is read-only and never accepts/declines an invite, creates a task, or fetches + remote content. Existing Safe Email content-mode semantics are preserved unchanged. + """ + kwargs = { + "mailbox": mailbox, "uid": uid, "account_id": account_id, "inspection": inspection, + "content_mode": content_mode, + "acknowledge_unsanitized_content_risk": acknowledge_unsanitized_content_risk, + } + try: + original = old_get_email(**kwargs) + except TypeError: + # Historical readers may not expose the later safe-inspection arguments. + original = old_get_email(mailbox=mailbox, uid=uid, account_id=account_id) + if not isinstance(original, Mapping): + return original + aid = _account_id(base, account_id) + analysis = _calendar_parts_from_detail( + core, mailbox=mailbox, uid=uid, account_id=aid, get_email_fn=old_get_email + ) + out = dict(original) + target = out.get("email") if isinstance(out.get("email"), Mapping) else out + if target is out: + out["calendar_analysis"] = analysis + else: + email_row = dict(target) + email_row["calendar_analysis"] = analysis + out["email"] = email_row + out["calendar_analysis_policy"] = { + "mandatory": True, "mime_and_headers": True, "network_access": False, + "automatic_rsvp": False, "automatic_task": False, "external_actions": False, + } + return out + + def email_search_status(account_id: str | None = None): + """Read-only. Return v9.9 email lexical/semantic index status for one account.""" + aid = _account_id(base, account_id) + result = get_email_index().status() + result.update({"account_id": aid, "imap_on_demand_enrichment": True, "external_remote_resource_fetch": False, "attachment_text": True}) + return result + + def search_emails_hybrid(query: str, mailbox: str = "INBOX", since_days: int = 90, limit: int = 20, + account_id: str | None = None, enrich_imap: bool = True, enrich_limit: int = 50): + """Read-only. Hybrid FTS5 + local semantic email search over subject/body and extractable attachments. + + When enrich_imap=true, Postmaster may fetch a bounded set of missing recent messages/attachments from + the selected IMAP mailbox before ranking. It never loads external URLs/resources contained in email. + """ + aid = _account_id(base, account_id) + enrichment = {"indexed": 0, "discovered": 0, "failures": [], "network_scope": "none"} + if enrich_imap: + enrichment = _enrich_email_index(base=base, core=core, index=get_email_index(), query=query, account_id=aid, + mailbox=mailbox, since_days=since_days, enrich_limit=enrich_limit) + result = get_email_index().search(query, account_id=aid, mailbox=mailbox, since_days=since_days, limit=limit) + result["enrichment"] = enrichment + result["external_remote_resource_fetch"] = False + return result + + def get_email_calendar_invites(mailbox: str, uid: str, account_id: str | None = None): + """Read-only. Analyze MIME/header/calendar material locally for invitations; never RSVP or communicate externally.""" + aid = _account_id(base, account_id) + result = _calendar_parts_from_detail(core, mailbox=mailbox, uid=uid, account_id=aid, get_email_fn=old_get_email) + result.update({"account_id": aid, "mailbox": mailbox, "uid": str(uid), "auto_rsvp": False, "auto_task": False}) + return result + + def send_job_calendar_invite(job_id: str, attendees: list[str] | None = None, account_id: str | None = None, + video_url: str | None = None, organizer_name: str | None = None, + track_opens: bool | None = None, tracking_disable_approval_id: str | None = None, + idempotency_key: str | None = None): + """WRITE ACTION. Explicitly send an RFC5545 calendar invitation for one stored passive task. + + create_job never sends. This tool is the explicit external action. A video URL already stored on the + task or supplied from another platform may be included. Tracking-OFF approval and recipient policy use + the normal email send boundary. + """ + job = core.get_job(job_id=job_id) + if not isinstance(job, Mapping): + return {"ok": False, "error": "Task not found"} + if isinstance(job.get("job"), Mapping): + job = job["job"] + payload = _as_dict(job.get("payload")) + cal = _as_dict(payload.get("calendar")) + recipients = _normalize_recipients(attendees or cal.get("attendees") or payload.get("attendees") or []) + if not recipients: + return {"ok": False, "error": "Calendar invitation requires at least one attendee"} + start_at, end_at = job_calendar_window(dict(job)) + title = str(cal.get("summary") or job.get("title") or "Scheduled task") + description = str(cal.get("description") or job.get("description") or "") + resolved_video = str(video_url or cal.get("video_url") or payload.get("video_url") or "").strip() or None + aid = _account_id(base, account_id) + account = _account(base, account_id) + organizer = str(account.get("email_address") or account.get("email") or "").strip() + if not organizer: + return {"ok": False, "error": "Selected account has no email address"} + uid = str(cal.get("uid") or f"postmaster-job-{job_id}@postmaster.local") + ics = build_calendar_request( + uid=uid, summary=title, start_utc=start_at, end_utc=end_at, organizer_email=organizer, + attendees=recipients, description=description, location=str(cal.get("location") or ""), + video_url=resolved_video, sequence=int(cal.get("sequence") or 0), + ) + attachment = { + "filename": "invite.ics", + "content_type": "text/calendar; method=REQUEST; charset=UTF-8", + "content_base64": base64.b64encode(ics.encode("utf-8")).decode("ascii"), + } + body = description or f"Calendar invitation: {title}" + if resolved_video: + body += f"\n\nVideo call: {resolved_video}" + return send_email( + to=recipients, subject=title, body=body, attachments=[attachment], track_opens=track_opens, + account_id=account_id, idempotency_key=idempotency_key or f"calendar-job:{job_id}:seq:{int(cal.get('sequence') or 0)}", + tracking_disable_approval_id=tracking_disable_approval_id, + ) + + def runtime_status(): + status = previous_runtime_status() + if not isinstance(status, dict): + status = {"ok": True} + status = dict(status) + status.update({ + "version_capability": "9.9.0", + "mcp_command_count_expected": MCP_COMMAND_COUNT_V990, + "mcp_command_count_delta_from_v980": MCP_EMAIL_CALENDAR_COMMANDS_V990 + MCP_WHATSAPP_COMMANDS_V990, + "tracking_disable_requires_fresh_chat_approval": True, + "tracking_disable_approval_one_use": True, + "google_image_proxy_likely_human_after_seconds": 60, + "proxy_attribution_metadata_reliable": False, + "subject_ascii_preferred": True, + "stored_file_link_preferred_over_attachment": True, + "calendar_invites": {"task_registry_passive": True, "explicit_send_only": True, "inbound_mime_local_analysis": True, "automatic_rsvp": False}, + "email_hybrid_search": {"fts5": True, "semantic": bool(embed_one) if semantic_resolved else False, "semantic_lazy": True, "attachments": True, "imap_on_demand": True, "external_resource_fetch": False}, + }) + return status + + # Replace existing names without changing their count; add four v9.9 email/calendar names. + replacements = ( + ("send_email", send_email, _annotations(read_only=False)), + ("reply_email", reply_email, _annotations(read_only=False)), + ("follow_up_email", follow_up_email, _annotations(read_only=False)), + ("list_open_events", list_open_events, _annotations(read_only=True, idempotent=True)), + ("get_email", get_email, _annotations(read_only=True, idempotent=True)), + ("runtime_status", runtime_status, _annotations(read_only=True, idempotent=True)), + ) + for name, fn, annotations in replacements: + core.mcp.remove_tool(name) + core.mcp.add_tool(fn, name=name, annotations=annotations) + setattr(core, name, fn) + setattr(base, name, fn) + additions = ( + ("email_search_status", email_search_status, _annotations(read_only=True, idempotent=True)), + ("search_emails_hybrid", search_emails_hybrid, _annotations(read_only=True, idempotent=True)), + ("get_email_calendar_invites", get_email_calendar_invites, _annotations(read_only=True, idempotent=True)), + ("send_job_calendar_invite", send_job_calendar_invite, _annotations(read_only=False)), + ) + for name, fn, annotations in additions: + core.mcp.add_tool(fn, name=name, annotations=annotations) + setattr(core, name, fn) + setattr(base, name, fn) + + base.tracking_approval_store_v990 = get_approval_store + base.email_search_index_v990 = get_email_index + return { + "approval_store": approval_store_obj, + "email_index": email_index_obj, + "approval_store_factory": get_approval_store, + "email_index_factory": get_email_index, + "runtime_status": runtime_status, + "mcp_command_count_expected": MCP_COMMAND_COUNT_V990, + } + + +__all__ = [ + "MCP_COMMAND_COUNT_V990", + "MCP_EMAIL_CALENDAR_COMMANDS_V990", + "MCP_WHATSAPP_COMMANDS_V990", + "install_runtime_v990", +] diff --git a/src/postmaster/tracking_policy_v990.py b/src/postmaster/tracking_policy_v990.py new file mode 100644 index 00000000..2d365172 --- /dev/null +++ b/src/postmaster/tracking_policy_v990.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +from contextlib import closing + +import hashlib +import json +import secrets +import sqlite3 +import threading +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any, Mapping + +TRACKING_DISABLE_APPROVAL_TTL_SECONDS = 300 +GOOGLE_PROXY_HUMAN_DELAY_SECONDS = 60 + + +class TrackingApprovalError(ValueError): + pass + + +def _utcnow() -> datetime: + return datetime.now(UTC) + + +def _iso(dt: datetime) -> str: + return dt.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +def _parse_iso(value: str) -> datetime: + text = str(value or "").strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + return datetime.fromisoformat(text).astimezone(UTC) + + +def _normalize(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, Mapping): + return {str(k): _normalize(v) for k, v in sorted(value.items(), key=lambda item: str(item[0]))} + if isinstance(value, (list, tuple, set)): + return [_normalize(v) for v in value] + return str(value) + + +def canonical_intent_hash(intent: Mapping[str, Any]) -> str: + encoded = json.dumps(_normalize(dict(intent)), ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def body_digest(text: str | None) -> str: + return hashlib.sha256((text or "").encode("utf-8")).hexdigest() + + +def attachment_intent_digest(attachments: Any) -> str: + """Hash stable attachment descriptors; never persist attachment bytes in approval storage.""" + return canonical_intent_hash({"attachments": _normalize(attachments or [])}) + + +def effective_tracking(track_opens: bool | None, *, account_tracking_default: bool) -> bool: + return bool(account_tracking_default if track_opens is None else track_opens) + + +def generated_subject_guidance(subject: str, *, explicitly_user_supplied: bool = False) -> dict[str, Any]: + """Soft policy only: generated subjects prefer ASCII; user text is never silently rewritten.""" + non_ascii = [ch for ch in str(subject) if ord(ch) > 127] + return { + "subject": subject, + "ascii": not non_ascii, + "preferred_ascii": True, + "should_suggest_ascii": bool(non_ascii) and not explicitly_user_supplied, + "preserve_user_supplied_unicode": bool(explicitly_user_supplied), + } + + +class TrackingApprovalStore: + """Persistent one-use approvals bound to an exact outbound intent. + + The preview id is a correlation handle, not reusable consent. A successful consume + marks the record used before the outbound action is delegated. A failed send therefore + needs a fresh chat approval, matching the requested "every time" boundary. + """ + + def __init__(self, db_path: str | Path = "/data/tracking-approvals-v990.db", *, ttl_seconds: int = TRACKING_DISABLE_APPROVAL_TTL_SECONDS): + self.db_path = str(db_path) + self.ttl_seconds = int(ttl_seconds) + self._lock = threading.RLock() + Path(self.db_path).parent.mkdir(parents=True, exist_ok=True) + self._init_db() + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path, timeout=10.0, isolation_level=None) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA foreign_keys=ON") + return conn + + def _init_db(self) -> None: + with closing(self._connect()) as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS tracking_disable_approvals ( + id TEXT PRIMARY KEY, + intent_hash TEXT NOT NULL, + operation TEXT NOT NULL, + created_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + consumed_at TEXT, + intent_summary TEXT NOT NULL DEFAULT '{}' + ) + """ + ) + conn.execute("CREATE INDEX IF NOT EXISTS idx_tracking_disable_expiry ON tracking_disable_approvals(expires_at)") + + def create(self, *, operation: str, intent: Mapping[str, Any], summary: Mapping[str, Any] | None = None) -> dict[str, Any]: + now = _utcnow() + approval_id = "trkoff_" + secrets.token_urlsafe(18) + digest = canonical_intent_hash(intent) + expires = now + timedelta(seconds=self.ttl_seconds) + safe_summary = json.dumps(_normalize(dict(summary or {})), ensure_ascii=True, sort_keys=True, separators=(",", ":")) + with self._lock, closing(self._connect()) as conn: + conn.execute("DELETE FROM tracking_disable_approvals WHERE expires_at < ?", (_iso(now),)) + conn.execute( + "INSERT INTO tracking_disable_approvals(id,intent_hash,operation,created_at,expires_at,intent_summary) VALUES(?,?,?,?,?,?)", + (approval_id, digest, str(operation), _iso(now), _iso(expires), safe_summary), + ) + return { + "ok": True, + "approval_required": True, + "approval_type": "tracking_disabled", + "preview_id": approval_id, + "intent_hash": digest, + "expires_at": _iso(expires), + "one_use": True, + "send_performed": False, + "message": "Fresh explicit chat approval is required because the effective outbound tracking state is OFF.", + "summary": _normalize(dict(summary or {})), + } + + def consume(self, approval_id: str, *, operation: str, intent: Mapping[str, Any]) -> dict[str, Any]: + aid = str(approval_id or "").strip() + if not aid: + raise TrackingApprovalError("tracking_disable_approval_id is required after explicit chat approval") + expected = canonical_intent_hash(intent) + now = _utcnow() + with self._lock, closing(self._connect()) as conn: + conn.execute("BEGIN IMMEDIATE") + row = conn.execute("SELECT * FROM tracking_disable_approvals WHERE id=?", (aid,)).fetchone() + if row is None: + conn.execute("ROLLBACK") + raise TrackingApprovalError("Tracking-disable approval does not exist or has already expired") + if row["consumed_at"]: + conn.execute("ROLLBACK") + raise TrackingApprovalError("Tracking-disable approval has already been consumed") + if _parse_iso(row["expires_at"]) < now: + conn.execute("DELETE FROM tracking_disable_approvals WHERE id=?", (aid,)) + conn.execute("COMMIT") + raise TrackingApprovalError("Tracking-disable approval expired; ask for fresh explicit chat approval") + if str(row["operation"]) != str(operation) or str(row["intent_hash"]) != expected: + conn.execute("ROLLBACK") + raise TrackingApprovalError("Tracking-disable approval does not match this exact outbound action") + conn.execute("UPDATE tracking_disable_approvals SET consumed_at=? WHERE id=?", (_iso(now), aid)) + conn.execute("COMMIT") + return {"ok": True, "consumed": True, "preview_id": aid, "intent_hash": expected, "consumed_at": _iso(now)} + + +def classify_open_event( + *, + sent_at: str | datetime | None, + opened_at: str | datetime | None, + user_agent: str = "", + client_source: str = "", + threshold_seconds: int = GOOGLE_PROXY_HUMAN_DELAY_SECONDS, +) -> dict[str, Any]: + ua = str(user_agent or "").lower() + source = str(client_source or "").lower() + google_proxy = "googleimageproxy" in ua or "google_image_proxy" in source or "gmail_image_proxy" in source + + def parse_dt(value: str | datetime | None) -> datetime | None: + if value is None: + return None + if isinstance(value, datetime): + dt = value + else: + text = str(value).strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + dt = datetime.fromisoformat(text) + except ValueError: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt.astimezone(UTC) + + sent = parse_dt(sent_at) + opened = parse_dt(opened_at) + delay: float | None = None + if sent is not None and opened is not None: + delay = max(0.0, (opened - sent).total_seconds()) + + metadata_note = "Proxy-derived IP/location/browser/OS metadata is unreliable and must not be treated as end-user attribution." + if google_proxy: + if delay is not None and delay > float(threshold_seconds): + return { + "classification": "likely_human_via_proxy", + "confidence": "medium", + "google_image_proxy": True, + "delay_seconds": delay, + "threshold_seconds": int(threshold_seconds), + "reason": f"Google Image Proxy fetch occurred more than {int(threshold_seconds)} seconds after delivery; timing is consistent with a later human-triggered open, while the fetch remains proxied.", + "metadata_reliability": "low", + "metadata_note": metadata_note, + } + return { + "classification": "likely_machine_or_proxy", + "confidence": "medium" if delay is not None else "low", + "google_image_proxy": True, + "delay_seconds": delay, + "threshold_seconds": int(threshold_seconds), + "reason": f"Google Image Proxy fetch occurred within {int(threshold_seconds)} seconds of delivery or delivery timing is unavailable; prefetch/automation remains plausible.", + "metadata_reliability": "low", + "metadata_note": metadata_note, + } + if "proxy" in ua or "proxy" in source or "scanner" in ua or "scanner" in source: + return { + "classification": "uncertain_proxy_or_scanner", + "confidence": "low", + "google_image_proxy": False, + "delay_seconds": delay, + "threshold_seconds": int(threshold_seconds), + "reason": "Proxy/scanner indicators are present and do not establish a human read.", + "metadata_reliability": "low", + "metadata_note": metadata_note, + } + return { + "classification": "human_or_unclassified", + "confidence": "low", + "google_image_proxy": False, + "delay_seconds": delay, + "threshold_seconds": int(threshold_seconds), + "reason": "No known image-proxy/scanner signature was identified; an image fetch alone still does not prove a human read.", + "metadata_reliability": "medium", + "metadata_note": "Observed network/client metadata may still reflect relays, VPNs or security software.", + } diff --git a/src/postmaster/webgui_whatsapp_v990.py b/src/postmaster/webgui_whatsapp_v990.py new file mode 100644 index 00000000..a045a885 --- /dev/null +++ b/src/postmaster/webgui_whatsapp_v990.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import base64 +import html +from io import BytesIO +from typing import Any, Mapping +from urllib.parse import quote + +from starlette.requests import Request +from starlette.responses import PlainTextResponse, RedirectResponse +from starlette.routing import Mount, Route + + +VIEW = "whatsapp" +LABEL = "WhatsApp" + + +def _local_qr_data_uri(payload: str) -> str | None: + """Render QR locally when qrcode is available; never calls an external service.""" + try: + import qrcode + import qrcode.image.svg + qr = qrcode.QRCode(error_correction=qrcode.constants.ERROR_CORRECT_M, border=4, box_size=8) + qr.add_data(str(payload)); qr.make(fit=True) + image = qr.make_image(image_factory=qrcode.image.svg.SvgPathImage) + buf = BytesIO(); image.save(buf) + return "data:image/svg+xml;base64," + base64.b64encode(buf.getvalue()).decode("ascii") + except Exception: + return None + + +def qr_renderer_status() -> dict[str, object]: + try: + import qrcode # noqa: F401 + available = True + except Exception: + available = False + return {"local": True, "remote_service": False, "available": available} + + +def render_whatsapp_panel(status: Mapping[str, Any], *, qr_payload: str | None = None) -> str: + network = status.get("network") if isinstance(status.get("network"), Mapping) else {} + connected = bool(network.get("connected")) + paired = bool(status.get("paired")) + state = "Connected" if connected else ("Paired / disconnected" if paired else "Not paired") + qr_uri = _local_qr_data_uri(str(qr_payload)) if qr_payload else None + if qr_payload and qr_uri: + qr_block = ( + '
' + '

Scan from WhatsApp → Linked devices. QR rendering is local and never uses a third-party endpoint.

' + ) + elif qr_payload: + qr_block = ( + '
Pairing QR renderer unavailable. ' + 'The transient pairing payload is retained encrypted server-side and is not exposed as plain text in this page.
' + ) + else: + qr_block = '

No active pairing QR. Start pairing explicitly to generate one.

' + return ( + '
' + '

WhatsApp

' + '

Clean-room companion session supervision.

v9.9.0
' + f'

Status: {html.escape(state)}

' + '

Reading messages here never sends read receipts. A read receipt is permitted only inside an explicit outbound reply flow.

' + f'{qr_block}' + '
' + ) + + +def _render_view(base: Any) -> str: + service = base.whatsapp_service_v990() + status = service.status() + qr = service.pairing_qr() + return '
' + render_whatsapp_panel(status, qr_payload=qr) + '
' + + +def _redirect(message: str = "") -> RedirectResponse: + suffix = ("?ui_view=whatsapp&flash=" + quote(message[:240])) if message else "?ui_view=whatsapp" + return RedirectResponse("/" + suffix + "#whatsapp", status_code=303) + + +async def _verified(base: Any, request: Request): + form, error = await base._verified_form(request) + if error: + return form, error + return form, None + + +async def start_pairing(base: Any, request: Request): + _form, error = await _verified(base, request) + if error: return error + try: + result = await base.whatsapp_service_v990().start_pairing() + message = "Pairing QR generated" if result.get("qr") else "Pairing started" + except Exception as exc: + message = f"{type(exc).__name__}: {exc}" + return _redirect(message) + + +async def reconnect(base: Any, request: Request): + _form, error = await _verified(base, request) + if error: return error + try: + result = await base.whatsapp_service_v990().reconnect() + message = "WhatsApp reconnected" if result.get("connected") or result.get("ok") else "Reconnect requested" + except Exception as exc: + message = f"{type(exc).__name__}: {exc}" + return _redirect(message) + + +def _insert_route(app: Any, route: Route) -> None: + for index, current in enumerate(app.router.routes): + if isinstance(current, Mount): + app.router.routes.insert(index, route); return + app.router.routes.append(route) + + +def _extend_nav(nav_module: Any) -> None: + groups = [] + found = False + for heading, links in nav_module.NAV_GROUPS: + mutable = list(links) + if any(item[0] == VIEW for item in mutable): found = True + if heading == "Communicate" and not found: + mutable.append((VIEW, LABEL, "WA")); found = True + groups.append((heading, tuple(mutable))) + if not found and groups: + heading, links = groups[-1]; groups[-1] = (heading, tuple(links) + ((VIEW, LABEL, "WA"),)) + nav_module.NAV_GROUPS = tuple(groups) + + +def install_webgui_whatsapp_v990(app: Any, base: Any, shell: Any, views: Any, nav_module: Any) -> None: + """Install the v9.9 WhatsApp supervision/pairing view into the existing lazy WebGUI shell.""" + old_render = shell.render_view + def render_with_whatsapp(bound_base: Any, core: Any, request: Request, view: str) -> str: + if view == VIEW: + return _render_view(bound_base) + return old_render(bound_base, core, request, view) + if VIEW not in shell.VIEWS: shell.VIEWS = tuple(shell.VIEWS) + (VIEW,) + if VIEW not in views.VIEWS: views.VIEWS = tuple(views.VIEWS) + (VIEW,) + shell.render_view = render_with_whatsapp + views.render_view = render_with_whatsapp + _extend_nav(nav_module) + + handlers = ( + ("/dashboard/whatsapp/pair", start_pairing, "v990_whatsapp_pair"), + ("/dashboard/whatsapp/reconnect", reconnect, "v990_whatsapp_reconnect"), + ) + existing = {getattr(route, "path", None) for route in app.router.routes} + for path, handler, name in handlers: + if path in existing: continue + async def endpoint(request: Request, _handler=handler): + if request.method != "POST": + return PlainTextResponse("Method Not Allowed", status_code=405, headers={"Allow": "POST"}) + return await _handler(base, request) + _insert_route(app, Route(path, endpoint, methods=["POST"], name=name)) + + +__all__ = [ + "VIEW", "LABEL", "qr_renderer_status", "render_whatsapp_panel", "start_pairing", "reconnect", "install_webgui_whatsapp_v990", +] diff --git a/src/postmaster/whatsapp_runtime_v990.py b/src/postmaster/whatsapp_runtime_v990.py new file mode 100644 index 00000000..f19e5177 --- /dev/null +++ b/src/postmaster/whatsapp_runtime_v990.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +from typing import Any, Callable + +from mcp.types import ToolAnnotations + +from .whatsapp_v990.service import WhatsAppService, WhatsAppServiceError + +MCP_WHATSAPP_COMMANDS_V990 = 8 + + +def _ann(*, read_only: bool, idempotent: bool = False): + return ToolAnnotations(read_only_hint=read_only, destructive_hint=False, idempotent_hint=idempotent, open_world_hint=False) + + +def install_whatsapp_runtime_v990( + base: Any, + core: Any, + previous_runtime_status: Callable[[], Any], + *, + service: WhatsAppService | None = None, + auth_db: str = "/data/whatsapp-v990-auth.db", + event_db: str = "/data/whatsapp-v990-events.db", + key_path: str = "/data/whatsapp-v990-auth.key", +) -> dict[str, Any]: + """Install the v9.9 WhatsApp MCP boundary. + + The service defaults to the clean-room current-protocol adapter, which is explicit-action + only: construction/status never opens the network. Pairing and reconnect require dedicated + actions. Direct/group text, media and Stored File handoff are implemented, while stable + readiness remains controlled-account gated. + """ + wa_obj = service + + def get_service() -> WhatsAppService: + nonlocal wa_obj + if wa_obj is None: + store_factory = getattr(base, "file_store", None) + file_store = store_factory() if callable(store_factory) else None + wa_obj = WhatsAppService.create( + auth_db=auth_db, + event_db=event_db, + key_path=key_path, + file_store=file_store, + ) + return wa_obj + + def safe_sync(fn, *args, **kwargs): + try: return fn(*args, **kwargs) + except (WhatsAppServiceError, ValueError) as exc: return {"ok": False, "error": str(exc)} + except Exception as exc: return {"ok": False, "error": f"{type(exc).__name__}: {exc}"} + + async def safe_async(fn, *args, **kwargs): + try: return await fn(*args, **kwargs) + except (WhatsAppServiceError, ValueError) as exc: return {"ok": False, "error": str(exc)} + except Exception as exc: return {"ok": False, "error": f"{type(exc).__name__}: {exc}"} + + def whatsapp_status(): + """Read-only. Show WhatsApp pairing/network/session status without exposing private keys or secrets.""" + return safe_sync(get_service().status) + + async def whatsapp_start_pairing(): + """WRITE ACTION. Explicitly start WhatsApp companion pairing and return the transient QR payload for WebGUI display.""" + return await safe_async(get_service().start_pairing) + + async def whatsapp_reconnect(): + """WRITE ACTION. Explicitly reconnect the persisted WhatsApp companion session; never auto-pairs a new device.""" + return await safe_async(get_service().reconnect) + + def whatsapp_list_messages(jid: str | None = None, limit: int = 100): + """Read-only. Read locally stored WhatsApp messages. Reading never emits a WhatsApp read receipt.""" + return safe_sync(get_service().list_messages, jid=jid, limit=limit) + + async def whatsapp_send_text(jid: str, text: str, reply_to_message_id: str | None = None): + """WRITE ACTION. Send one WhatsApp text/group message. A read receipt may be emitted only when this explicit send is a reply.""" + return await safe_async(get_service().send_text, jid=jid, text=text, reply_to_message_id=reply_to_message_id) + + async def whatsapp_send_media(jid: str, stored_file_id: str, caption: str = "", reply_to_message_id: str | None = None): + """WRITE ACTION. Send WhatsApp media from a Postmaster Stored File id; do not route raw attachment Base64 through the MCP call.""" + return await safe_async(get_service().send_media, jid=jid, stored_file_id=stored_file_id, caption=caption, reply_to_message_id=reply_to_message_id) + + async def whatsapp_list_groups(): + """Read-only. List groups visible to the paired WhatsApp companion session.""" + return await safe_async(get_service().list_groups) + + def whatsapp_list_receipts(limit: int = 200): + """Read-only. List remote receipts observed by Postmaster plus outbound-reply receipts emitted under the asymmetric policy.""" + return safe_sync(get_service().list_receipts, limit=limit) + + tools = ( + ("whatsapp_status", whatsapp_status, _ann(read_only=True, idempotent=True)), + ("whatsapp_start_pairing", whatsapp_start_pairing, _ann(read_only=False)), + ("whatsapp_reconnect", whatsapp_reconnect, _ann(read_only=False)), + ("whatsapp_list_messages", whatsapp_list_messages, _ann(read_only=True, idempotent=True)), + ("whatsapp_send_text", whatsapp_send_text, _ann(read_only=False)), + ("whatsapp_send_media", whatsapp_send_media, _ann(read_only=False)), + ("whatsapp_list_groups", whatsapp_list_groups, _ann(read_only=True, idempotent=True)), + ("whatsapp_list_receipts", whatsapp_list_receipts, _ann(read_only=True, idempotent=True)), + ) + for name, fn, annotations in tools: + try: core.mcp.remove_tool(name) + except Exception: pass + core.mcp.add_tool(fn, name=name, annotations=annotations) + setattr(core, name, fn); setattr(base, name, fn) + + old_status = previous_runtime_status + def runtime_status(): + status = old_status() + status = dict(status) if isinstance(status, dict) else {"ok": True} + wa_status = wa_obj.status() if wa_obj is not None else {"network": {"configured": False, "connected": False}, "paired": False} + status["whatsapp"] = { + "clean_room_python": True, + "tool_count": MCP_WHATSAPP_COMMANDS_V990, + "network_adapter_configured": bool(wa_status.get("network", {}).get("configured")), + "paired": bool(wa_status.get("paired")), + "read_receipts_asymmetric": True, + "local_read_emits_receipt": False, + "auth_encrypted_at_rest": True, + "private_material_exposed": False, + "implementation_complete": True, + "controlled_acceptance_required": True, + "protocol_interop_verified": False, + "signal_multidevice_verified": False, + "controlled_account_interop_verified": False, + } + return status + + # Runtime status is a replacement, not an additional command. + try: core.mcp.remove_tool("runtime_status") + except Exception: pass + core.mcp.add_tool(runtime_status, name="runtime_status", annotations=_ann(read_only=True, idempotent=True)) + setattr(core, "runtime_status", runtime_status); setattr(base, "runtime_status", runtime_status) + base.whatsapp_service_v990 = get_service + return {"service": wa_obj, "service_factory": get_service, "runtime_status": runtime_status, "tool_count": MCP_WHATSAPP_COMMANDS_V990} + + +__all__ = ["MCP_WHATSAPP_COMMANDS_V990", "install_whatsapp_runtime_v990"] diff --git a/src/postmaster/whatsapp_v990/__init__.py b/src/postmaster/whatsapp_v990/__init__.py new file mode 100644 index 00000000..3d04a345 --- /dev/null +++ b/src/postmaster/whatsapp_v990/__init__.py @@ -0,0 +1,4 @@ +"""Postmaster v9.9 clean-room WhatsApp Web protocol package.""" + +from .handshake import * # noqa: F401,F403 +from .pairing import * # noqa: F401,F403 diff --git a/src/postmaster/whatsapp_v990/adapter.py b/src/postmaster/whatsapp_v990/adapter.py new file mode 100644 index 00000000..741aeebf --- /dev/null +++ b/src/postmaster/whatsapp_v990/adapter.py @@ -0,0 +1,1564 @@ +from __future__ import annotations + +"""Current-protocol clean-room WhatsApp companion adapter. + +Pairing and reconnect are explicit actions. Constructing the adapter never opens the network. +Direct and group Signal/media paths are implemented clean-room and exercised offline; stable +acceptance remains fail-closed until controlled-account interoperability is demonstrated. +""" + +import asyncio +import base64 +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +import os +import secrets +from typing import Any, Awaitable, Callable, Mapping + +from .adv import encode_signed_device_identity, verify_and_sign_pair_success_identity +from .binary import BinaryNode, BinaryNodeCodec +from .cert import verify_noise_certificate_chain +from .client_payload import RegistrationKeys, build_login_payload, build_registration_payload +from .crypto import CurveKeyPair, WhatsAppNoiseXX, frame_noise_payload, generate_curve_keypair, split_noise_frames +from .handshake import decode_handshake, encode_client_finish, encode_client_hello +from .groups import ( + build_group_metadata_query, + build_participating_groups_query, + parse_group_metadata_response, + parse_participating_groups, +) +from .jid import parse_jid, same_user, transfer_device +from .media import ( + build_media_conn_query, + decode_media_descriptor, + download_media, + encode_media_message, + infer_media_type, + parse_media_conn, + upload_media_bytes, +) +from .messages import ( + build_ack_stanza, + build_direct_message_stanza, + build_group_message_stanza, + build_read_receipt, + decode_text_message, + encode_device_sent_message, + encode_reply_text_message, + encode_text_message, + encrypted_participant_node, + generate_message_id_v2, + pad_random_max16, + participant_hash_v2, + unpad_random_max16, +) +from .pairing import build_pairing_qr_data +from .signal_keys import SignedPreKey, generate_registration_id, generate_signed_pre_key +from .signal_server import ( + build_prekey_count_query, + build_prekey_upload, + build_session_query, + parse_prekey_count, + parse_session_bundles, +) +from .signal_session import EncryptedSignalSessionStore, decrypt_prekey_message, initialize_outgoing_session +from .sender_key import ( + EncryptedSenderKeyStore, + decode_sender_key_distribution_message, + encode_sender_key_distribution_message, +) +from .signal_wire import PreKeyWhisperMessageV3 +from .store import EncryptedAuthStore +from .usync import build_device_query, parse_device_result +from .tokens import CURRENT_TOKEN_TABLE +from .websocket_driver import WebSocketDriverConfig, open_whatsapp_websocket + + +class CurrentProtocolAdapterError(RuntimeError): + pass + + +@dataclass(slots=True) +class ProtocolCredentials: + noise: CurveKeyPair + identity: CurveKeyPair + signed_pre_key: SignedPreKey + registration_id: int + adv_secret_b64: str + registered: bool = False + jid: str | None = None + lid: str | None = None + platform: str | None = None + account_identity_b64: str | None = None + next_pre_key_id: int = 1 + first_unuploaded_pre_key_id: int = 1 + + def to_json(self) -> dict[str, Any]: + def b64(value: bytes) -> str: + return base64.b64encode(value).decode("ascii") + return { + "noise_private": b64(self.noise.private), + "noise_public": b64(self.noise.public), + "identity_private": b64(self.identity.private), + "identity_public": b64(self.identity.public), + "signed_pre_key_private": b64(self.signed_pre_key.key_pair.private), + "signed_pre_key_public": b64(self.signed_pre_key.key_pair.public), + "signed_pre_key_signature": b64(self.signed_pre_key.signature), + "signed_pre_key_id": self.signed_pre_key.key_id, + "registration_id": self.registration_id, + "adv_secret_b64": self.adv_secret_b64, + "registered": self.registered, + "jid": self.jid, + "lid": self.lid, + "platform": self.platform, + "account_identity_b64": self.account_identity_b64, + "next_pre_key_id": self.next_pre_key_id, + "first_unuploaded_pre_key_id": self.first_unuploaded_pre_key_id, + } + + @classmethod + def from_json(cls, value: Mapping[str, Any]) -> "ProtocolCredentials": + def raw(name: str) -> bytes: + try: + return base64.b64decode(str(value[name]), validate=True) + except Exception as exc: + raise CurrentProtocolAdapterError(f"Stored WhatsApp credential {name} is invalid") from exc + noise = CurveKeyPair(raw("noise_private"), raw("noise_public")) + identity = CurveKeyPair(raw("identity_private"), raw("identity_public")) + spk_pair = CurveKeyPair(raw("signed_pre_key_private"), raw("signed_pre_key_public")) + spk = SignedPreKey(int(value["signed_pre_key_id"]), spk_pair, raw("signed_pre_key_signature")) + return cls( + noise=noise, + identity=identity, + signed_pre_key=spk, + registration_id=int(value["registration_id"]), + adv_secret_b64=str(value["adv_secret_b64"]), + registered=bool(value.get("registered")), + jid=str(value.get("jid")) if value.get("jid") else None, + lid=str(value.get("lid")) if value.get("lid") else None, + platform=str(value.get("platform")) if value.get("platform") else None, + account_identity_b64=str(value.get("account_identity_b64")) if value.get("account_identity_b64") else None, + next_pre_key_id=int(value.get("next_pre_key_id", 1)), + first_unuploaded_pre_key_id=int(value.get("first_unuploaded_pre_key_id", 1)), + ) + + +class WhatsAppWireSession: + def __init__(self, ws: Any, transport: Any, *, codec: BinaryNodeCodec | None = None): + self.ws = ws + self.transport = transport + self.codec = codec or BinaryNodeCodec(CURRENT_TOKEN_TABLE) + self.pending = b"" + self.nodes: list[BinaryNode] = [] + self.closed = False + self._send_lock = asyncio.Lock() + self._query_lock = asyncio.Lock() + self._recv_lock = asyncio.Lock() + self._dispatch_task: asyncio.Task | None = None + self._dispatch_handler: Callable[[BinaryNode], Awaitable[None]] | None = None + self._waiters: dict[tuple[str, str | None], asyncio.Future] = {} + self._unsolicited: asyncio.Queue[BinaryNode] = asyncio.Queue() + + @classmethod + async def open( + cls, + *, + noise_static: CurveKeyPair, + client_payload: bytes, + websocket_config: WebSocketDriverConfig | None = None, + ) -> "WhatsAppWireSession": + ephemeral = generate_curve_keypair() + noise = WhatsAppNoiseXX(ephemeral) + ws = await open_whatsapp_websocket(websocket_config or WebSocketDriverConfig(prefer_native=True)) + try: + await ws.send(frame_noise_payload(encode_client_hello(ephemeral.public), intro=b"WA\x06\x03")) + pending = b"" + frames: list[bytes] = [] + while not frames: + value = await asyncio.wait_for(ws.recv(), timeout=15) + if isinstance(value, str): + raise CurrentProtocolAdapterError("WhatsApp returned text during Noise handshake") + frames, pending = split_noise_frames(pending + bytes(value)) + if len(frames) != 1: + raise CurrentProtocolAdapterError(f"Expected one Noise ServerHello frame, got {len(frames)}") + server = decode_handshake(frames[0]).server_hello + if server is None: + raise CurrentProtocolAdapterError("WhatsApp did not return Noise ServerHello") + encrypted_static, cert_chain = noise.process_server_hello( + server_ephemeral=server.ephemeral, + encrypted_static=server.static, + encrypted_payload=server.payload, + noise_static=noise_static, + ) + verify_noise_certificate_chain(cert_chain) + encrypted_payload = noise.encrypt(client_payload) + await ws.send(frame_noise_payload(encode_client_finish(encrypted_static, encrypted_payload))) + session = cls(ws, noise.finish()) + session.pending = pending + return session + except Exception: + await ws.close() + raise + + @property + def dispatching(self) -> bool: + return self._dispatch_task is not None and not self._dispatch_task.done() + + async def _recv_direct_node(self, *, timeout: float = 30.0) -> BinaryNode: + async with self._recv_lock: + if self.nodes: + return self.nodes.pop(0) + while not self.closed: + value = await asyncio.wait_for(self.ws.recv(), timeout=timeout) + if isinstance(value, str): + raise CurrentProtocolAdapterError("WhatsApp returned an unexpected text WebSocket message") + frames, self.pending = split_noise_frames(self.pending + bytes(value)) + for frame in frames: + plaintext = self.transport.decrypt(frame) + self.nodes.append(self.codec.decode(plaintext)) + if self.nodes: + return self.nodes.pop(0) + raise CurrentProtocolAdapterError("WhatsApp wire session is closed") + + async def recv_node(self, *, timeout: float = 30.0) -> BinaryNode: + if self.dispatching: + return await asyncio.wait_for(self._unsolicited.get(), timeout=timeout) + return await self._recv_direct_node(timeout=timeout) + + async def send_node(self, node: BinaryNode) -> None: + wire = self.codec.encode(node) + encrypted = self.transport.encrypt(wire) + async with self._send_lock: + await self.ws.send(frame_noise_payload(encrypted)) + + async def _dispatch_loop(self) -> None: + try: + while not self.closed: + try: + node = await self._recv_direct_node(timeout=60.0) + except TimeoutError: + continue + stanza_id = str(node.attrs.get("id") or "") + matched = False + if stanza_id: + for key in ((stanza_id, node.tag), (stanza_id, None)): + future = self._waiters.pop(key, None) + if future is not None and not future.done(): + future.set_result(node) + matched = True + break + if matched: + continue + if self._dispatch_handler is not None: + await self._dispatch_handler(node) + else: + self._unsolicited.put_nowait(node) + except asyncio.CancelledError: + return + except Exception as exc: + error = CurrentProtocolAdapterError(f"WhatsApp receive dispatcher stopped: {type(exc).__name__}: {exc}") + for future in list(self._waiters.values()): + if not future.done(): + future.set_exception(error) + self._waiters.clear() + self.closed = True + + def start_dispatcher(self, handler: Callable[[BinaryNode], Awaitable[None]] | None = None) -> None: + if self.closed: + raise CurrentProtocolAdapterError("Cannot start dispatcher on a closed WhatsApp session") + if self.dispatching: + if handler is not None: + self._dispatch_handler = handler + return + self._dispatch_handler = handler + self._dispatch_task = asyncio.create_task(self._dispatch_loop()) + + async def send_and_wait( + self, + node: BinaryNode, + *, + response_tag: str | None = None, + timeout: float = 30.0, + ) -> BinaryNode: + stanza_id = str(node.attrs.get("id") or "").strip() + if not stanza_id: + raise CurrentProtocolAdapterError("Correlated WhatsApp stanza must have an id") + if not self.dispatching: + deferred: list[BinaryNode] = [] + await self.send_node(node) + try: + async with asyncio.timeout(timeout): + while True: + current = await self._recv_direct_node(timeout=timeout) + if current.attrs.get("id") == stanza_id and (response_tag is None or current.tag == response_tag): + return current + deferred.append(current) + finally: + if deferred: + self.nodes = deferred + self.nodes + + key = (stanza_id, response_tag) + if key in self._waiters: + raise CurrentProtocolAdapterError(f"Duplicate correlated WhatsApp waiter for {stanza_id}") + future = asyncio.get_running_loop().create_future() + self._waiters[key] = future + try: + await self.send_node(node) + return await asyncio.wait_for(asyncio.shield(future), timeout=timeout) + finally: + current = self._waiters.get(key) + if current is future: + self._waiters.pop(key, None) + + async def query(self, node: BinaryNode, *, timeout: float = 30.0) -> BinaryNode: + if node.tag != "iq": + raise CurrentProtocolAdapterError("Correlated WhatsApp query must be an iq node") + if not node.attrs.get("id"): + node.attrs["id"] = "pm-" + secrets.token_hex(8) + async with self._query_lock: + return await self.send_and_wait(node, response_tag="iq", timeout=timeout) + + async def close(self) -> None: + self.closed = True + task, self._dispatch_task = self._dispatch_task, None + if task is not None and task is not asyncio.current_task() and not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + for future in list(self._waiters.values()): + if not future.done(): + future.cancel() + self._waiters.clear() + await self.ws.close() + + +SessionOpener = Callable[..., Awaitable[WhatsAppWireSession]] + + +def _new_credentials() -> ProtocolCredentials: + identity = generate_curve_keypair() + return ProtocolCredentials( + noise=generate_curve_keypair(), + identity=identity, + signed_pre_key=generate_signed_pre_key(identity, 1), + registration_id=generate_registration_id(), + adv_secret_b64=base64.b64encode(os.urandom(32)).decode("ascii"), + ) + + +def _expires(seconds: int) -> str: + return (datetime.now(UTC) + timedelta(seconds=seconds)).isoformat().replace("+00:00", "Z") + + +class CurrentProtocolAdapter: + def __init__( + self, + auth: EncryptedAuthStore, + *, + session_opener: SessionOpener = WhatsAppWireSession.open, + on_message: Callable[..., Any] | None = None, + on_receipt: Callable[..., Any] | None = None, + file_store: Any | None = None, + file_owner_id: str = "default", + file_project_id: str | None = None, + ): + self.auth = auth + self.session_opener = session_opener + self.on_message = on_message + self.on_receipt = on_receipt + self.file_store = file_store + self.file_owner_id = str(file_owner_id or "default").strip() or "default" + self.file_project_id = str(file_project_id).strip() if file_project_id else None + self.session: WhatsAppWireSession | None = None + self._pair_task: asyncio.Task | None = None + self._qr_task: asyncio.Task | None = None + self._keepalive_task: asyncio.Task | None = None + self._last_error: str | None = None + self._connected = False + + def _load(self) -> ProtocolCredentials | None: + value = self.auth.get_json("protocol", "credentials") + return ProtocolCredentials.from_json(value) if value else None + + def _save(self, creds: ProtocolCredentials) -> None: + self.auth.put_json("protocol", "credentials", creds.to_json()) + + def _store_pre_key(self, key_id: int, pair: CurveKeyPair) -> None: + self.auth.put_json("signal-pre-key", str(int(key_id)), { + "private": base64.b64encode(pair.private).decode("ascii"), + "public": base64.b64encode(pair.public).decode("ascii"), + }) + + def _load_pre_key(self, key_id: int) -> CurveKeyPair | None: + value = self.auth.get_json("signal-pre-key", str(int(key_id))) + if not value: + return None + try: + private = base64.b64decode(str(value["private"]), validate=True) + public = base64.b64decode(str(value["public"]), validate=True) + except Exception as exc: + raise CurrentProtocolAdapterError("Stored Signal pre-key is corrupt") from exc + if len(private) != 32 or len(public) != 32: + raise CurrentProtocolAdapterError("Stored Signal pre-key has invalid length") + return CurveKeyPair(private, public) + + def _pending_pre_keys(self, creds: ProtocolCredentials, count: int) -> dict[int, CurveKeyPair]: + start = int(creds.first_unuploaded_pre_key_id) + target = start + int(count) + while creds.next_pre_key_id < target: + key_id = int(creds.next_pre_key_id) + self._store_pre_key(key_id, generate_curve_keypair()) + creds.next_pre_key_id = key_id + 1 + self._save(creds) + result: dict[int, CurveKeyPair] = {} + for key_id in range(start, target): + pair = self._load_pre_key(key_id) + if pair is None: + pair = generate_curve_keypair() + self._store_pre_key(key_id, pair) + result[key_id] = pair + return result + + async def _ensure_server_pre_keys(self, session: WhatsAppWireSession, creds: ProtocolCredentials) -> int: + count_response = await session.query(build_prekey_count_query(), timeout=30) + server_count = parse_prekey_count(count_response) + upload_count = 812 if server_count == 0 else (5 if server_count <= 5 else 0) + if upload_count == 0: + return server_count + start = int(creds.first_unuploaded_pre_key_id) + keys = self._pending_pre_keys(creds, upload_count) + response = await session.query( + build_prekey_upload( + registration_id=creds.registration_id, + identity_public=creds.identity.public, + signed_pre_key=creds.signed_pre_key, + pre_keys=keys, + ), + timeout=60, + ) + if response.attrs.get("type") == "error" or response.child("error") is not None: + raise CurrentProtocolAdapterError("WhatsApp rejected Signal pre-key upload") + creds.first_unuploaded_pre_key_id = start + upload_count + self._save(creds) + return server_count + upload_count + + async def _stop_tasks(self) -> None: + current = asyncio.current_task() + for task in (self._pair_task, self._qr_task, self._keepalive_task): + if task is not None and task is not current and not task.done(): + task.cancel() + self._pair_task = None + self._qr_task = None + self._keepalive_task = None + if self.session is not None: + try: + await self.session.close() + except Exception: + pass + self.session = None + self._connected = False + + async def close(self) -> None: + await self._stop_tasks() + + async def _registration_session(self, creds: ProtocolCredentials) -> WhatsAppWireSession: + payload = build_registration_payload( + RegistrationKeys( + registration_id=creds.registration_id, + identity_public_key=creds.identity.public, + signed_pre_key=creds.signed_pre_key, + ) + ) + return await self.session_opener(noise_static=creds.noise, client_payload=payload) + + async def _login_session(self, creds: ProtocolCredentials) -> WhatsAppWireSession: + if not creds.jid: + raise CurrentProtocolAdapterError("Stored WhatsApp session has no paired JID") + jid = parse_jid(creds.jid) + try: + username = int(jid.user) + except ValueError as exc: + raise CurrentProtocolAdapterError("Stored WhatsApp JID user is not numeric") from exc + payload = build_login_payload(username=username, device=int(jid.device or 0)) + return await self.session_opener(noise_static=creds.noise, client_payload=payload) + + @staticmethod + def _pair_device(node: BinaryNode) -> BinaryNode | None: + return node.child("pair-device") if node.tag == "iq" else None + + async def start_pairing(self) -> Mapping[str, Any]: + await self._stop_tasks() + creds = _new_credentials() + self._save(creds) + session = await self._registration_session(creds) + self.session = session + + pair_iq: BinaryNode | None = None + pair_device: BinaryNode | None = None + for _ in range(24): + node = await session.recv_node(timeout=20) + candidate = self._pair_device(node) + if candidate is not None: + pair_iq, pair_device = node, candidate + break + if pair_iq is None or pair_device is None: + await self._stop_tasks() + raise CurrentProtocolAdapterError("WhatsApp did not provide pair-device QR references") + + stanza_id = pair_iq.attrs.get("id") + if not stanza_id: + await self._stop_tasks() + raise CurrentProtocolAdapterError("pair-device stanza has no id") + await session.send_node(BinaryNode("iq", {"to": "s.whatsapp.net", "type": "result", "id": stanza_id})) + + refs: list[str] = [] + for node in pair_device.children("ref"): + value = node.content + if isinstance(value, bytes): + try: + text = value.decode("utf-8") + except UnicodeDecodeError: + continue + elif isinstance(value, str): + text = value + else: + continue + if text: + refs.append(text) + if not refs: + await self._stop_tasks() + raise CurrentProtocolAdapterError("pair-device stanza contained no usable QR refs") + + def qr_for(ref: str) -> str: + return build_pairing_qr_data( + ref, + creds.noise.public, + creds.identity.public, + creds.adv_secret_b64, + os_name="Linux", + browser_name="Chrome", + ) + + first_qr = qr_for(refs[0]) + self.auth.put_json("runtime", "pairing", {"qr": first_qr, "expires_at": _expires(60)}) + self._pair_task = asyncio.create_task(self._pair_success_loop(creds, session)) + self._qr_task = asyncio.create_task(self._rotate_qr(refs[1:], qr_for)) + self._keepalive_task = asyncio.create_task(self._keepalive(session)) + return { + "ok": True, + "qr": first_qr, + "expires_at": _expires(60), + "paired": False, + "network_adapter": "clean_room_current_protocol", + "private_material_exposed": False, + } + + async def _rotate_qr(self, refs: list[str], qr_for: Callable[[str], str]) -> None: + try: + await asyncio.sleep(60) + for ref in refs: + qr = qr_for(ref) + self.auth.put_json("runtime", "pairing", {"qr": qr, "expires_at": _expires(20)}) + await asyncio.sleep(20) + except asyncio.CancelledError: + return + + async def _pair_success_loop(self, creds: ProtocolCredentials, session: WhatsAppWireSession) -> None: + try: + while True: + node = await session.recv_node(timeout=90) + if node.tag in {"failure", "stream:error"}: + raise CurrentProtocolAdapterError(f"WhatsApp pairing failed with {node.tag}") + pair_success = node.child("pair-success") if node.tag == "iq" else None + if pair_success is None: + continue + identity_node = pair_success.child("device-identity") + device_node = pair_success.child("device") + platform_node = pair_success.child("platform") + if identity_node is None or not isinstance(identity_node.content, bytes) or device_node is None: + raise CurrentProtocolAdapterError("pair-success missing device identity/device") + verified = verify_and_sign_pair_success_identity( + identity_node.content, + adv_secret_key=creds.adv_secret_b64, + signed_identity_key=creds.identity, + ) + jid = device_node.attrs.get("jid") + lid = device_node.attrs.get("lid") + if not jid: + raise CurrentProtocolAdapterError("pair-success device has no JID") + key_index = verified.device_identity.key_index + if key_index is None: + raise CurrentProtocolAdapterError("pair-success device identity has no key index") + reply = BinaryNode( + "iq", + {"to": "s.whatsapp.net", "type": "result", "id": node.attrs.get("id", "")}, + [ + BinaryNode( + "pair-device-sign", + {}, + [ + BinaryNode( + "device-identity", + {"key-index": str(key_index)}, + verified.encoded_for_reply, + ) + ], + ) + ], + ) + await session.send_node(reply) + creds.registered = True + creds.jid = jid + creds.lid = lid + creds.platform = platform_node.attrs.get("name") if platform_node is not None else None + creds.account_identity_b64 = base64.b64encode( + encode_signed_device_identity(verified.signed_identity, include_signature_key=True) + ).decode("ascii") + self._save(creds) + self.auth.put_json("runtime", "identity", {"jid": jid, "lid": lid, "platform": creds.platform}) + self.auth.delete("runtime", "pairing") + self._connected = False + # WhatsApp normally asks the newly paired companion to restart/login. + return + except asyncio.CancelledError: + return + except Exception as exc: + self._last_error = f"{type(exc).__name__}: {exc}" + self._connected = False + + async def _keepalive(self, session: WhatsAppWireSession) -> None: + try: + while not session.closed: + await asyncio.sleep(20) + await session.send_node( + BinaryNode( + "iq", + {"to": "s.whatsapp.net", "type": "get", "xmlns": "w:p", "id": "pm-" + secrets.token_hex(6)}, + [BinaryNode("ping")], + ) + ) + except asyncio.CancelledError: + return + except Exception as exc: + self._last_error = f"{type(exc).__name__}: {exc}" + self._connected = False + + async def reconnect(self) -> Mapping[str, Any]: + await self._stop_tasks() + creds = self._load() + if creds is None or not creds.registered or not creds.jid: + raise CurrentProtocolAdapterError("No paired WhatsApp session is stored") + session = await self._login_session(creds) + self.session = session + for _ in range(32): + node = await session.recv_node(timeout=20) + if node.tag == "success": + pre_key_count = await self._ensure_server_pre_keys(session, creds) + self._connected = True + session.start_dispatcher(self._handle_unsolicited) + self._keepalive_task = asyncio.create_task(self._keepalive(session)) + return {"ok": True, "connected": True, "paired": True, "server_pre_key_count": pre_key_count} + if node.tag in {"failure", "stream:error"}: + await self._stop_tasks() + raise CurrentProtocolAdapterError(f"WhatsApp login failed with {node.tag}") + await self._stop_tasks() + raise CurrentProtocolAdapterError("WhatsApp login did not reach success state") + + async def _emit_callback(self, callback: Callable[..., Any] | None, **kwargs: Any) -> None: + if callback is None: + return + result = callback(**kwargs) + if asyncio.iscoroutine(result): + await result + + def _incoming_decryption_jid(self, node: BinaryNode) -> str: + author = str(node.attrs.get("participant") or node.attrs.get("from") or "").strip() + if not author: + raise CurrentProtocolAdapterError("Incoming WhatsApp message has no sender") + parsed = parse_jid(author) + if parsed.is_lid: + return str(parsed) + alt = ( + node.attrs.get("participant_lid") + or node.attrs.get("sender_lid") + or node.attrs.get("peer_recipient_lid") + ) + if alt: + try: + return str(transfer_device(parsed, parse_jid(str(alt)))) + except ValueError: + pass + return str(parsed) + + async def _decrypt_pairwise_message_proto( + self, + node: BinaryNode, + creds: ProtocolCredentials, + *, + e2e_type: str, + ciphertext: bytes, + ) -> bytes: + decryption_jid = self._incoming_decryption_jid(node) + sessions = EncryptedSignalSessionStore(self.auth) + if e2e_type == "pkmsg": + envelope = PreKeyWhisperMessageV3.parse(ciphertext) + one_time = self._load_pre_key(envelope.pre_key_id) if envelope.pre_key_id is not None else None + signal, plaintext, consumed_pre_key = decrypt_prekey_message( + ciphertext, + our_identity=creds.identity, + our_signed_pre_key=creds.signed_pre_key, + our_one_time_pre_key=one_time, + our_registration_id=creds.registration_id, + ) + sessions.save(decryption_jid, signal) + if consumed_pre_key is not None: + self.auth.delete("signal-pre-key", str(int(consumed_pre_key))) + elif e2e_type == "msg": + signal = sessions.load(decryption_jid) + if signal is None: + raise CurrentProtocolAdapterError( + f"No Signal session for incoming WhatsApp sender {decryption_jid}" + ) + plaintext = signal.decrypt_signal(ciphertext) + sessions.save(decryption_jid, signal) + else: + raise CurrentProtocolAdapterError( + f"Unsupported WhatsApp pairwise E2E type {e2e_type!r}" + ) + return unpad_random_max16(plaintext) + + async def _materialize_incoming_content( + self, + message_proto: bytes, + *, + message_id: str | None, + ) -> tuple[str, str, str | None]: + text = decode_text_message(message_proto) + if text is not None: + return "text", text, None + + descriptor = decode_media_descriptor(message_proto) + if descriptor is None: + return "unknown", "", None + if self.file_store is None: + raise CurrentProtocolAdapterError( + "Postmaster Stored File store is not configured for incoming WhatsApp media" + ) + + payload = await download_media(descriptor) + suffix = { + "image": ".jpg", + "video": ".mp4", + "audio": ".ogg", + "document": ".bin", + "sticker": ".webp", + }.get(descriptor.media_type, ".bin") + raw_name = str(descriptor.filename or "").replace("\\", "/").split("/")[-1].strip() + if ( + not raw_name + or raw_name in {".", ".."} + or "\x00" in raw_name + or len(raw_name) > 200 + ): + token = str(message_id or secrets.token_hex(8)).replace("/", "_")[:80] + raw_name = f"whatsapp-{token}{suffix}" + try: + info = self.file_store.save_bytes( + owner_id=self.file_owner_id, + project_id=self.file_project_id, + filename=raw_name, + data=payload, + media_type=descriptor.mimetype, + description="Inbound WhatsApp media", + tags=["whatsapp", "inbound"], + ) + except Exception as exc: + raise CurrentProtocolAdapterError( + f"Unable to save inbound WhatsApp media as Stored File: {exc}" + ) from exc + file_id = str(info.get("id") or "").strip() + if not file_id: + raise CurrentProtocolAdapterError( + "Stored File save returned no file id for inbound WhatsApp media" + ) + return "media", str(descriptor.caption or ""), file_id + + async def _handle_incoming_group_message( + self, + node: BinaryNode, + creds: ProtocolCredentials, + ) -> None: + group_id = str(parse_jid(str(node.attrs.get("from") or ""))) + if not parse_jid(group_id).is_group: + raise CurrentProtocolAdapterError("Group message handler requires a g.us sender") + author_raw = str(node.attrs.get("participant") or "").strip() + if not author_raw: + raise CurrentProtocolAdapterError("Incoming WhatsApp group message has no participant") + author_jid = self._incoming_decryption_jid(node) + sender_keys = EncryptedSenderKeyStore(self.auth) + + kind = "unknown" + text = "" + stored_file_id: str | None = None + saw_group_ciphertext = False + for enc in node.children("enc"): + if not isinstance(enc.content, (bytes, bytearray, memoryview)): + continue + e2e_type = str(enc.attrs.get("type") or "") + ciphertext = bytes(enc.content) + if e2e_type in {"pkmsg", "msg"}: + message_proto = await self._decrypt_pairwise_message_proto( + node, + creds, + e2e_type=e2e_type, + ciphertext=ciphertext, + ) + distribution = decode_sender_key_distribution_message(message_proto) + if distribution is not None: + distribution_group, distribution_wire = distribution + if str(parse_jid(distribution_group)) != group_id: + raise CurrentProtocolAdapterError( + "Sender-key distribution group does not match incoming group stanza" + ) + sender_keys.process_distribution( + group_id, + author_jid, + distribution_wire, + ) + else: + kind, text, stored_file_id = await self._materialize_incoming_content( + message_proto, + message_id=str(node.attrs.get("id") or "") or None, + ) + continue + if e2e_type == "skmsg": + plaintext = sender_keys.decrypt(group_id, author_jid, ciphertext) + message_proto = unpad_random_max16(plaintext) + kind, text, stored_file_id = await self._materialize_incoming_content( + message_proto, + message_id=str(node.attrs.get("id") or "") or None, + ) + saw_group_ciphertext = True + continue + raise CurrentProtocolAdapterError( + f"Unsupported WhatsApp group E2E type {e2e_type!r}" + ) + + if not saw_group_ciphertext: + # A standalone sender-key distribution updates local crypto state but is not a + # user-visible chat message. + return + + direction = ( + "out" + if same_user(author_raw, creds.jid) + or (creds.lid and same_user(author_raw, creds.lid)) + else "in" + ) + await self._emit_callback( + self.on_message, + message_id=str(node.attrs.get("id") or "") or None, + jid=group_id, + direction=direction, + kind=kind, + text=text, + stored_file_id=stored_file_id, + reply_to_message_id=None, + ) + + async def _handle_incoming_message(self, node: BinaryNode) -> None: + creds = self._load() + if creds is None or not creds.registered or not creds.jid: + raise CurrentProtocolAdapterError("Incoming WhatsApp message arrived without paired credentials") + sender = str(node.attrs.get("participant") or node.attrs.get("from") or "").strip() + if not sender: + raise CurrentProtocolAdapterError("Incoming WhatsApp message has no sender") + if parse_jid(str(node.attrs.get("from") or sender)).is_group: + await self._handle_incoming_group_message(node, creds) + return + enc = next( + ( + child for child in node.children("enc") + if isinstance(child.content, (bytes, bytearray, memoryview)) + ), + None, + ) + if enc is None: + raise CurrentProtocolAdapterError("Incoming WhatsApp direct message has no encrypted payload") + ciphertext = bytes(enc.content) + e2e_type = str(enc.attrs.get("type") or "") + message_proto = await self._decrypt_pairwise_message_proto( + node, + creds, + e2e_type=e2e_type, + ciphertext=ciphertext, + ) + kind, text, stored_file_id = await self._materialize_incoming_content( + message_proto, + message_id=str(node.attrs.get("id") or "") or None, + ) + direction = "out" if same_user(sender, creds.jid) or (creds.lid and same_user(sender, creds.lid)) else "in" + if direction == "out" and node.attrs.get("recipient"): + conversation = str(parse_jid(str(node.attrs["recipient"])).normalized_user()) + else: + conversation = str(parse_jid(sender).normalized_user()) + await self._emit_callback( + self.on_message, + message_id=str(node.attrs.get("id") or "") or None, + jid=conversation, + direction=direction, + kind=kind, + text=text, + stored_file_id=stored_file_id, + reply_to_message_id=None, + ) + + async def _handle_unsolicited(self, node: BinaryNode) -> None: + wire = self.session + creds = self._load() + if wire is None: + return + if node.tag == "message": + try: + await self._handle_incoming_message(node) + await wire.send_node(build_ack_stanza(node, me_id=creds.jid if creds else None)) + except Exception as exc: + self._last_error = f"{type(exc).__name__}: {exc}" + try: + await wire.send_node(build_ack_stanza(node, me_id=creds.jid if creds else None, error_code=500)) + except Exception: + pass + return + if node.tag == "receipt": + await self._emit_callback( + self.on_receipt, + message_id=str(node.attrs.get("id") or "") or None, + jid=str(node.attrs.get("from") or node.attrs.get("participant") or "") or None, + receipt_type=str(node.attrs.get("type") or "delivery"), + source="remote", + ) + if node.attrs.get("id") and node.attrs.get("from"): + await wire.send_node(build_ack_stanza(node, me_id=creds.jid if creds else None)) + return + if node.tag == "notification": + if node.attrs.get("id") and node.attrs.get("from"): + await wire.send_node(build_ack_stanza(node, me_id=creds.jid if creds else None)) + return + if node.tag in {"failure", "stream:error"}: + self._last_error = f"WhatsApp unsolicited {node.tag}" + self._connected = False + + def _require_live_session(self) -> tuple[WhatsAppWireSession, ProtocolCredentials]: + creds = self._load() + if creds is None or not creds.registered or not creds.jid: + raise CurrentProtocolAdapterError("No paired WhatsApp session is stored") + if not self._connected or self.session is None or self.session.closed: + raise CurrentProtocolAdapterError("WhatsApp companion session is not connected; reconnect explicitly first") + return self.session, creds + + async def _ensure_signal_sessions( + self, + wire: WhatsAppWireSession, + creds: ProtocolCredentials, + addresses: list[str], + ) -> EncryptedSignalSessionStore: + store = EncryptedSignalSessionStore(self.auth) + missing = [address for address in dict.fromkeys(addresses) if store.load(address) is None] + if not missing: + return store + response = await wire.query(build_session_query(missing), timeout=30) + bundles = parse_session_bundles(response) + for address in missing: + bundle = bundles.get(address) + if bundle is None: + # Servers should echo the requested device JID exactly. Keep a conservative + # normalized fallback for equivalent textual encodings only. + target = str(parse_jid(address)) + bundle = next( + (value for key, value in bundles.items() if str(parse_jid(key)) == target), + None, + ) + if bundle is None: + raise CurrentProtocolAdapterError(f"WhatsApp returned no Signal pre-key bundle for {address}") + signal = initialize_outgoing_session( + our_identity=creds.identity, + bundle=bundle, + our_registration_id=creds.registration_id, + ) + store.save(address, signal) + return store + + async def _wait_message_ack( + self, + wire: WhatsAppWireSession, + message_id: str, + *, + timeout: float = 30.0, + ) -> BinaryNode: + deferred: list[BinaryNode] = [] + try: + async with asyncio.timeout(timeout): + while True: + node = await wire.recv_node(timeout=timeout) + if node.tag == "ack" and node.attrs.get("id") == message_id: + if node.attrs.get("error") not in (None, "", "0"): + raise CurrentProtocolAdapterError( + f"WhatsApp rejected message {message_id} with error {node.attrs.get('error')}" + ) + return node + if node.tag in {"failure", "stream:error"}: + raise CurrentProtocolAdapterError(f"WhatsApp send failed with {node.tag}") + deferred.append(node) + finally: + if deferred: + wire.nodes = deferred + wire.nodes + + async def _send_group_payload( + self, + *, + wire: WhatsAppWireSession, + creds: ProtocolCredentials, + destination: Any, + message_proto: bytes, + message_type: str, + enc_attrs: Mapping[str, str] | None = None, + ) -> Mapping[str, Any]: + metadata_response = await wire.query(build_group_metadata_query(str(destination)), timeout=30) + metadata = parse_group_metadata_response(metadata_response) + addressing_mode = str(metadata.get("addressing_mode") or "pn") + prefer_lid = addressing_mode == "lid" + + participant_ids: list[str] = [] + for participant in metadata.get("participants") or []: + if not isinstance(participant, Mapping): + continue + selected = ( + participant.get("lid") + if prefer_lid and participant.get("lid") + else participant.get("id") + ) + if selected: + participant_ids.append(str(selected)) + + own_pn = parse_jid(creds.jid) + own_lid = parse_jid(creds.lid) if creds.lid else None + sender_identity = ( + own_lid.normalized_user() + if prefer_lid and own_lid is not None + else own_pn.normalized_user() + ) + participant_ids.append(str(sender_identity)) + participant_ids = list(dict.fromkeys(participant_ids)) + if not participant_ids: + raise CurrentProtocolAdapterError("WhatsApp group metadata contained no participants") + + usync = await wire.query( + build_device_query(participant_ids, context="message"), + timeout=30, + ) + discovered = parse_device_result( + usync, + own_jid=creds.lid if prefer_lid and creds.lid else creds.jid, + prefer_lid=prefer_lid, + ) + + own_users = {own_pn.user} + if own_lid is not None: + own_users.add(own_lid.user) + own_device = int(own_pn.device or 0) + targets = [] + for target in discovered: + parsed = parse_jid(target.jid) + if target.hosted or int(target.device or 0) == 99: + continue + if parsed.user in own_users and int(parsed.device or 0) == own_device: + continue + targets.append(target) + if not targets: + raise CurrentProtocolAdapterError( + "WhatsApp USync returned no devices for group sender-key distribution" + ) + + group_id = str(destination) + sender_keys = EncryptedSenderKeyStore(self.auth) + distribution = sender_keys.distribution(group_id, str(sender_identity)) + memory = self.auth.get_json("group-sender-key-memory", group_id) or {} + remembered = {str(value) for value in (memory.get("devices") or []) if str(value)} + recipients = [target for target in targets if target.jid not in remembered] + + participant_nodes: list[BinaryNode] = [] + include_device_identity = False + if recipients: + sessions = await self._ensure_signal_sessions( + wire, + creds, + [target.jid for target in recipients], + ) + distribution_proto = pad_random_max16( + encode_sender_key_distribution_message(group_id, distribution) + ) + for target in recipients: + signal = sessions.load(target.jid) + if signal is None: + raise CurrentProtocolAdapterError( + f"Signal session disappeared for group sender-key recipient {target.jid}" + ) + ciphertext_type, ciphertext = signal.encrypt(distribution_proto) + sessions.save(target.jid, signal) + participant_nodes.append( + encrypted_participant_node( + target.jid, + ciphertext_type=ciphertext_type, + ciphertext=ciphertext, + ) + ) + include_device_identity = include_device_identity or ciphertext_type == "pkmsg" + + device_identity: bytes | None = None + if include_device_identity: + if not creds.account_identity_b64: + raise CurrentProtocolAdapterError( + "Paired WhatsApp credentials have no signed device identity" + ) + try: + device_identity = base64.b64decode(creds.account_identity_b64, validate=True) + except Exception as exc: + raise CurrentProtocolAdapterError( + "Stored WhatsApp signed device identity is corrupt" + ) from exc + + plaintext = pad_random_max16(bytes(message_proto)) + group_ciphertext = sender_keys.encrypt(group_id, str(sender_identity), plaintext) + message_id = generate_message_id_v2(creds.jid) + top_attrs: dict[str, str] = {} + ephemeral = metadata.get("ephemeral_duration") + if isinstance(ephemeral, int) and ephemeral > 0: + top_attrs["expiration"] = str(ephemeral) + stanza = build_group_message_stanza( + destination_jid=group_id, + message_id=message_id, + sender_key_ciphertext=group_ciphertext, + sender_key_recipients=participant_nodes, + device_identity=device_identity, + message_type=message_type, + addressing_mode=addressing_mode, + additional_attrs=top_attrs, + enc_attrs=dict(enc_attrs or {}), + ) + ack = await wire.send_and_wait(stanza, response_tag="ack", timeout=30) + if ack.attrs.get("error") not in (None, "", "0"): + raise CurrentProtocolAdapterError( + f"WhatsApp rejected group message {message_id} with error {ack.attrs.get('error')}" + ) + + if recipients: + remembered.update(target.jid for target in recipients) + self.auth.put_json( + "group-sender-key-memory", + group_id, + {"devices": sorted(remembered)}, + ) + return { + "ok": True, + "message_id": message_id, + "to": group_id, + "server_ack": True, + "ack_class": ack.attrs.get("class"), + "group": True, + "addressing_mode": addressing_mode, + "group_participant_count": len(metadata.get("participants") or []), + "device_fanout": len(targets), + "sender_key_recipients": len(recipients), + "used_prekey_message": include_device_identity, + "read_receipt_emitted": False, + } + + async def _send_group_text( + self, + *, + wire: WhatsAppWireSession, + creds: ProtocolCredentials, + destination: Any, + text: str, + reply_to_message_id: str | None, + emit_read_receipt: bool, + ) -> Mapping[str, Any]: + if reply_to_message_id or emit_read_receipt: + raise CurrentProtocolAdapterError( + "WhatsApp group reply/read-receipt flow is not acceptance-complete; send a non-reply group message" + ) + return await self._send_group_payload( + wire=wire, + creds=creds, + destination=destination, + message_proto=encode_text_message(text), + message_type="text", + ) + + async def send_text( + self, + *, + jid: str, + text: str, + reply_to_message_id: str | None = None, + emit_read_receipt: bool = False, + ) -> Mapping[str, Any]: + """Send a direct or group text using the current Signal multi-device protocol. + + Group messages use SenderKey v3 and distribute the sender key pairwise only to devices + that have not already received it. Group reply/read-receipt behavior remains fail-closed. + """ + wire, creds = self._require_live_session() + destination = parse_jid(jid).normalized_user() + if destination.is_group: + value = str(text) + if not value: + raise CurrentProtocolAdapterError("WhatsApp text cannot be empty") + return await self._send_group_text( + wire=wire, + creds=creds, + destination=destination, + text=value, + reply_to_message_id=reply_to_message_id, + emit_read_receipt=emit_read_receipt, + ) + if destination.is_broadcast or destination.is_newsletter: + raise CurrentProtocolAdapterError("WhatsApp broadcast/newsletter send is not supported by this v9.9 direct-send path") + value = str(text) + if not value: + raise CurrentProtocolAdapterError("WhatsApp text cannot be empty") + + own_pn = parse_jid(creds.jid) + own_lid = parse_jid(creds.lid) if creds.lid else None + sender_identity = own_lid.normalized_user() if destination.is_lid and own_lid is not None else own_pn.normalized_user() + usync = await wire.query( + build_device_query([str(sender_identity), str(destination)], context="message"), + timeout=30, + ) + discovered = parse_device_result( + usync, + own_jid=creds.lid if destination.is_lid and creds.lid else creds.jid, + prefer_lid=True, + ) + + own_users = {own_pn.user} + if own_lid is not None: + own_users.add(own_lid.user) + own_device = int(own_pn.device or 0) + targets = [] + for target in discovered: + parsed = parse_jid(target.jid) + # LID migration can change the user part, so parse_device_result's own_jid filter + # alone cannot always recognize the current companion. Exclude it explicitly. + if parsed.user in own_users and int(parsed.device or 0) == own_device: + continue + targets.append(target) + if not targets: + raise CurrentProtocolAdapterError("WhatsApp USync returned no target devices for direct send") + + target_jids = [target.jid for target in targets] + sessions = await self._ensure_signal_sessions(wire, creds, target_jids) + message_id = generate_message_id_v2(creds.jid) + if reply_to_message_id: + plain_proto = encode_reply_text_message( + value, + stanza_id=str(reply_to_message_id), + participant=str(destination), + remote_jid=str(destination), + ) + else: + plain_proto = encode_text_message(value) + phash = participant_hash_v2(target_jids) + # WhatsApp pads each top-level WAProto.Message before Signal encryption. The nested + # DeviceSentMessage contains the unpadded original message and is itself padded once. + plain = pad_random_max16(plain_proto) + dsm = pad_random_max16(encode_device_sent_message(str(destination), plain_proto, phash=phash)) + + participants: list[BinaryNode] = [] + include_device_identity = False + own_count = 0 + remote_count = 0 + for target in targets: + target_jid = target.jid + target_user = parse_jid(target_jid).user + is_own = target_user in own_users + signal = sessions.load(target_jid) + if signal is None: + raise CurrentProtocolAdapterError(f"Signal session disappeared for {target_jid}") + ciphertext_type, ciphertext = signal.encrypt(dsm if is_own else plain) + sessions.save(target_jid, signal) + participants.append( + encrypted_participant_node( + target_jid, + ciphertext_type=ciphertext_type, + ciphertext=ciphertext, + ) + ) + include_device_identity = include_device_identity or ciphertext_type == "pkmsg" + if is_own: + own_count += 1 + else: + remote_count += 1 + + device_identity: bytes | None = None + if include_device_identity: + if not creds.account_identity_b64: + raise CurrentProtocolAdapterError("Paired WhatsApp credentials have no signed device identity") + try: + device_identity = base64.b64decode(creds.account_identity_b64, validate=True) + except Exception as exc: + raise CurrentProtocolAdapterError("Stored WhatsApp signed device identity is corrupt") from exc + + stanza = build_direct_message_stanza( + destination_jid=str(destination), + message_id=message_id, + participants=participants, + device_identity=device_identity, + message_type="text", + ) + ack = await wire.send_and_wait(stanza, response_tag="ack", timeout=30) + if ack.attrs.get("error") not in (None, "", "0"): + raise CurrentProtocolAdapterError( + f"WhatsApp rejected message {message_id} with error {ack.attrs.get('error')}" + ) + + receipt_emitted = False + if emit_read_receipt and reply_to_message_id: + await wire.send_node( + build_read_receipt( + destination_jid=str(destination), + message_id=str(reply_to_message_id), + ) + ) + receipt_emitted = True + + return { + "ok": True, + "message_id": message_id, + "to": str(destination), + "server_ack": True, + "ack_class": ack.attrs.get("class"), + "device_fanout": len(participants), + "own_device_targets": own_count, + "remote_device_targets": remote_count, + "used_prekey_message": include_device_identity, + "read_receipt_emitted": receipt_emitted, + } + + async def send_media( + self, + *, + jid: str, + stored_file_id: str, + caption: str = "", + reply_to_message_id: str | None = None, + emit_read_receipt: bool = False, + ) -> Mapping[str, Any]: + """Encrypt/upload one Stored File and relay its media protobuf through direct or group E2E. + + Raw file bytes stay server-side. Groups use the same SenderKey v3 pipeline as group text; + group reply/read-receipt behavior remains fail-closed pending controlled acceptance. + """ + wire, creds = self._require_live_session() + destination = parse_jid(jid).normalized_user() + if destination.is_group and (reply_to_message_id or emit_read_receipt): + raise CurrentProtocolAdapterError( + "WhatsApp group media reply/read-receipt flow is not acceptance-complete" + ) + if destination.is_broadcast or destination.is_newsletter: + raise CurrentProtocolAdapterError("WhatsApp broadcast/newsletter media send is not supported") + fid = str(stored_file_id or "").strip() + if not fid: + raise CurrentProtocolAdapterError("stored_file_id is required") + if self.file_store is None: + raise CurrentProtocolAdapterError("Postmaster Stored File store is not configured for WhatsApp media") + try: + info, data = self.file_store.raw_bytes(fid) + except Exception as exc: + raise CurrentProtocolAdapterError(f"Unable to read Postmaster Stored File {fid}: {exc}") from exc + filename = str(info.get("filename") or "file") + mimetype = str(info.get("media_type") or "application/octet-stream") + media_type = infer_media_type(mimetype=mimetype, filename=filename) + + conn_response = await wire.query(build_media_conn_query(), timeout=30) + media_conn = parse_media_conn(conn_response) + upload, _encrypted = await upload_media_bytes( + data, + media_type=media_type, + mimetype=mimetype, + filename=filename, + caption=str(caption or ""), + media_conn=media_conn, + ) + plain_proto = encode_media_message(upload) + + if destination.is_group: + result = dict( + await self._send_group_payload( + wire=wire, + creds=creds, + destination=destination, + message_proto=plain_proto, + message_type="media", + enc_attrs={"mediatype": media_type}, + ) + ) + result.update( + { + "stored_file_id": fid, + "media_type": media_type, + "mimetype": mimetype, + "filename": filename, + "file_size": len(data), + } + ) + return result + + own_pn = parse_jid(creds.jid) + own_lid = parse_jid(creds.lid) if creds.lid else None + sender_identity = own_lid.normalized_user() if destination.is_lid and own_lid is not None else own_pn.normalized_user() + usync = await wire.query( + build_device_query([str(sender_identity), str(destination)], context="message"), + timeout=30, + ) + discovered = parse_device_result( + usync, + own_jid=creds.lid if destination.is_lid and creds.lid else creds.jid, + prefer_lid=True, + ) + + own_users = {own_pn.user} + if own_lid is not None: + own_users.add(own_lid.user) + own_device = int(own_pn.device or 0) + targets = [] + for target in discovered: + parsed = parse_jid(target.jid) + if parsed.user in own_users and int(parsed.device or 0) == own_device: + continue + targets.append(target) + if not targets: + raise CurrentProtocolAdapterError("WhatsApp USync returned no target devices for media send") + + target_jids = [target.jid for target in targets] + sessions = await self._ensure_signal_sessions(wire, creds, target_jids) + message_id = generate_message_id_v2(creds.jid) + phash = participant_hash_v2(target_jids) + plain = pad_random_max16(plain_proto) + dsm = pad_random_max16(encode_device_sent_message(str(destination), plain_proto, phash=phash)) + + participants: list[BinaryNode] = [] + include_device_identity = False + own_count = 0 + remote_count = 0 + for target in targets: + target_jid = target.jid + target_user = parse_jid(target_jid).user + is_own = target_user in own_users + signal = sessions.load(target_jid) + if signal is None: + raise CurrentProtocolAdapterError(f"Signal session disappeared for {target_jid}") + ciphertext_type, ciphertext = signal.encrypt(dsm if is_own else plain) + sessions.save(target_jid, signal) + participants.append( + encrypted_participant_node( + target_jid, + ciphertext_type=ciphertext_type, + ciphertext=ciphertext, + ) + ) + include_device_identity = include_device_identity or ciphertext_type == "pkmsg" + own_count += 1 if is_own else 0 + remote_count += 0 if is_own else 1 + + device_identity: bytes | None = None + if include_device_identity: + if not creds.account_identity_b64: + raise CurrentProtocolAdapterError("Paired WhatsApp credentials have no signed device identity") + try: + device_identity = base64.b64decode(creds.account_identity_b64, validate=True) + except Exception as exc: + raise CurrentProtocolAdapterError("Stored WhatsApp signed device identity is corrupt") from exc + + stanza = build_direct_message_stanza( + destination_jid=str(destination), + message_id=message_id, + participants=participants, + device_identity=device_identity, + message_type="media", + additional_attrs={"mediatype": media_type}, + ) + ack = await wire.send_and_wait(stanza, response_tag="ack", timeout=30) + if ack.attrs.get("error") not in (None, "", "0"): + raise CurrentProtocolAdapterError( + f"WhatsApp rejected media message {message_id} with error {ack.attrs.get('error')}" + ) + + receipt_emitted = False + if emit_read_receipt and reply_to_message_id: + await wire.send_node( + build_read_receipt( + destination_jid=str(destination), + message_id=str(reply_to_message_id), + ) + ) + receipt_emitted = True + + return { + "ok": True, + "message_id": message_id, + "to": str(destination), + "stored_file_id": fid, + "media_type": media_type, + "mimetype": mimetype, + "filename": filename, + "file_size": len(data), + "server_ack": True, + "ack_class": ack.attrs.get("class"), + "device_fanout": len(participants), + "own_device_targets": own_count, + "remote_device_targets": remote_count, + "used_prekey_message": include_device_identity, + "read_receipt_emitted": receipt_emitted, + } + + async def list_groups(self) -> list[Mapping[str, Any]]: + """Read participating group metadata through the current w:g2 protocol. + + Group text/media SenderKey send and receive are implemented separately; stable readiness + remains controlled-account gated. + """ + wire, _creds = self._require_live_session() + response = await wire.query(build_participating_groups_query(), timeout=30) + return parse_participating_groups(response) + + def status(self) -> Mapping[str, Any]: + creds = self._load() + pairing = self.auth.get_json("runtime", "pairing") or {} + return { + "configured": True, + "connected": self._connected, + "paired": bool(creds and creds.registered and creds.jid), + "pairing_pending": bool(pairing.get("qr")), + "native_websocket": True, + "implementation_complete": True, + "controlled_acceptance_required": True, + "signal_send_implemented": True, + "signal_send_ready": False, + "media_implemented": True, + "media_ready": False, + "group_listing_implemented": True, + "group_text_send_implemented": True, + "group_text_receive_implemented": True, + "group_media_send_implemented": True, + "media_receive_to_stored_file_implemented": True, + "groups_ready": False, + "last_error": self._last_error, + } + + +__all__ = [ + "CurrentProtocolAdapterError", "ProtocolCredentials", "WhatsAppWireSession", "CurrentProtocolAdapter" +] diff --git a/src/postmaster/whatsapp_v990/adv.py b/src/postmaster/whatsapp_v990/adv.py new file mode 100644 index 00000000..b7cb9c5f --- /dev/null +++ b/src/postmaster/whatsapp_v990/adv.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +"""WhatsApp ADV device-identity verification/signing for companion pairing. + +The wire shapes are small protobuf messages from the public WhatsApp Web schema. This module +does not perform network I/O and never logs ADV secrets or private keys. +""" + +from dataclasses import dataclass, replace +import base64 +import hashlib +import hmac + +from .crypto import CurveKeyPair, xeddsa_sign, xeddsa_verify +from .proto import ProtoError, decode_fields, field_bytes, field_varint + +ADV_E2EE = 0 +ADV_HOSTED = 1 + +WA_ADV_ACCOUNT_SIG_PREFIX = bytes((6, 0)) +WA_ADV_DEVICE_SIG_PREFIX = bytes((6, 1)) +WA_ADV_HOSTED_ACCOUNT_SIG_PREFIX = bytes((6, 5)) +WA_ADV_HOSTED_DEVICE_SIG_PREFIX = bytes((6, 6)) + + +class ADVError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class ADVDeviceIdentity: + raw_id: int | None = None + timestamp: int | None = None + key_index: int | None = None + account_type: int | None = None + device_type: int | None = None + + +@dataclass(frozen=True, slots=True) +class ADVSignedDeviceIdentity: + details: bytes + account_signature_key: bytes | None + account_signature: bytes | None + device_signature: bytes | None = None + + +@dataclass(frozen=True, slots=True) +class ADVSignedDeviceIdentityHMAC: + details: bytes + hmac_value: bytes + account_type: int | None = None + + +@dataclass(frozen=True, slots=True) +class VerifiedPairingIdentity: + signed_identity: ADVSignedDeviceIdentity + device_identity: ADVDeviceIdentity + encoded_for_reply: bytes + account_signature_verified: bool = True + hmac_verified: bool = True + + +def _map(raw: bytes) -> dict[int, list[int | bytes]]: + try: + fields = decode_fields(bytes(raw)) + except ProtoError as exc: + raise ADVError("Invalid ADV protobuf") from exc + out: dict[int, list[int | bytes]] = {} + for field in fields: + out.setdefault(field.number, []).append(field.value) + return out + + +def _last_bytes(fields: dict[int, list[int | bytes]], number: int, *, required: bool = False) -> bytes | None: + values = fields.get(number) or [] + value = values[-1] if values else None + if value is None: + if required: + raise ADVError(f"Missing ADV bytes field {number}") + return None + if not isinstance(value, bytes): + raise ADVError(f"ADV field {number} must be bytes") + return bytes(value) + + +def _last_int(fields: dict[int, list[int | bytes]], number: int) -> int | None: + values = fields.get(number) or [] + value = values[-1] if values else None + if value is None: + return None + if not isinstance(value, int): + raise ADVError(f"ADV field {number} must be varint") + return int(value) + + +def decode_adv_device_identity(raw: bytes) -> ADVDeviceIdentity: + fields = _map(raw) + return ADVDeviceIdentity( + raw_id=_last_int(fields, 1), + timestamp=_last_int(fields, 2), + key_index=_last_int(fields, 3), + account_type=_last_int(fields, 4), + device_type=_last_int(fields, 5), + ) + + +def encode_adv_device_identity(value: ADVDeviceIdentity) -> bytes: + parts: list[bytes] = [] + for number, item in ( + (1, value.raw_id), + (2, value.timestamp), + (3, value.key_index), + (4, value.account_type), + (5, value.device_type), + ): + if item is not None: + parts.append(field_varint(number, int(item))) + return b"".join(parts) + + +def decode_signed_device_identity(raw: bytes) -> ADVSignedDeviceIdentity: + fields = _map(raw) + details = _last_bytes(fields, 1, required=True) + assert details is not None + account_key = _last_bytes(fields, 2) + account_signature = _last_bytes(fields, 3) + device_signature = _last_bytes(fields, 4) + if account_key is not None and len(account_key) not in (32, 33): + raise ADVError("ADV account signature key must be Curve25519 public-key length") + for label, signature in (("account", account_signature), ("device", device_signature)): + if signature is not None and len(signature) != 64: + raise ADVError(f"ADV {label} signature must be 64 bytes") + return ADVSignedDeviceIdentity(details, account_key, account_signature, device_signature) + + +def encode_signed_device_identity(value: ADVSignedDeviceIdentity, *, include_signature_key: bool) -> bytes: + parts = [field_bytes(1, value.details)] + if include_signature_key and value.account_signature_key: + parts.append(field_bytes(2, value.account_signature_key)) + if value.account_signature: + parts.append(field_bytes(3, value.account_signature)) + if value.device_signature: + parts.append(field_bytes(4, value.device_signature)) + return b"".join(parts) + + +def decode_signed_device_identity_hmac(raw: bytes) -> ADVSignedDeviceIdentityHMAC: + fields = _map(raw) + details = _last_bytes(fields, 1, required=True) + mac = _last_bytes(fields, 2, required=True) + assert details is not None and mac is not None + if len(mac) != 32: + raise ADVError("ADV device-identity HMAC must be 32 bytes") + return ADVSignedDeviceIdentityHMAC(details=details, hmac_value=mac, account_type=_last_int(fields, 3)) + + +def encode_signed_device_identity_hmac(value: ADVSignedDeviceIdentityHMAC) -> bytes: + parts = [field_bytes(1, value.details), field_bytes(2, value.hmac_value)] + if value.account_type is not None: + parts.append(field_varint(3, int(value.account_type))) + return b"".join(parts) + + +def _adv_secret(value: bytes | str) -> bytes: + if isinstance(value, str): + try: + raw = base64.b64decode(value, validate=True) + except Exception as exc: + raise ADVError("ADV secret must be valid Base64") from exc + else: + raw = bytes(value) + if len(raw) != 32: + raise ADVError("ADV secret must decode to 32 bytes") + return raw + + +def verify_and_sign_pair_success_identity( + device_identity_hmac_blob: bytes, + *, + adv_secret_key: bytes | str, + signed_identity_key: CurveKeyPair, + random64: bytes | None = None, +) -> VerifiedPairingIdentity: + """Verify a pair-success device identity and add our companion device signature. + + The returned encoding deliberately omits accountSignatureKey, matching the pair-device-sign + reply contract. The account key remains available in the structured return value for Signal + identity persistence. + """ + outer = decode_signed_device_identity_hmac(device_identity_hmac_blob) + secret = _adv_secret(adv_secret_key) + hmac_prefix = WA_ADV_HOSTED_ACCOUNT_SIG_PREFIX if outer.account_type == ADV_HOSTED else b"" + expected_mac = hmac.new(secret, hmac_prefix + outer.details, hashlib.sha256).digest() + if not hmac.compare_digest(expected_mac, outer.hmac_value): + raise ADVError("Invalid ADV account HMAC") + + account = decode_signed_device_identity(outer.details) + if account.account_signature_key is None or account.account_signature is None: + raise ADVError("ADV signed account identity is missing signature material") + + device_details = decode_adv_device_identity(account.details) + account_prefix = ( + WA_ADV_HOSTED_ACCOUNT_SIG_PREFIX + if device_details.device_type == ADV_HOSTED + else WA_ADV_ACCOUNT_SIG_PREFIX + ) + account_message = account_prefix + account.details + signed_identity_key.public + if not xeddsa_verify(account.account_signature_key, account_message, account.account_signature): + raise ADVError("Invalid ADV account signature") + + device_message = ( + WA_ADV_DEVICE_SIG_PREFIX + + account.details + + signed_identity_key.public + + account.account_signature_key + ) + device_signature = xeddsa_sign(signed_identity_key.private, device_message, random64=random64) + signed = replace(account, device_signature=device_signature) + encoded = encode_signed_device_identity(signed, include_signature_key=False) + return VerifiedPairingIdentity( + signed_identity=signed, + device_identity=device_details, + encoded_for_reply=encoded, + ) + + +__all__ = [ + "ADV_E2EE", "ADV_HOSTED", + "WA_ADV_ACCOUNT_SIG_PREFIX", "WA_ADV_DEVICE_SIG_PREFIX", + "WA_ADV_HOSTED_ACCOUNT_SIG_PREFIX", "WA_ADV_HOSTED_DEVICE_SIG_PREFIX", + "ADVError", "ADVDeviceIdentity", "ADVSignedDeviceIdentity", "ADVSignedDeviceIdentityHMAC", + "VerifiedPairingIdentity", "decode_adv_device_identity", "encode_adv_device_identity", + "decode_signed_device_identity", "encode_signed_device_identity", + "decode_signed_device_identity_hmac", "encode_signed_device_identity_hmac", + "verify_and_sign_pair_success_identity", +] diff --git a/src/postmaster/whatsapp_v990/binary.py b/src/postmaster/whatsapp_v990/binary.py new file mode 100644 index 00000000..e58364df --- /dev/null +++ b/src/postmaster/whatsapp_v990/binary.py @@ -0,0 +1,325 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +import zlib +from typing import Iterable, Mapping, Sequence + +from .jid import JID, parse_jid + + +class BinaryNodeError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class BinaryTags: + LIST_EMPTY: int = 0 + DICTIONARY_0: int = 236 + DICTIONARY_1: int = 237 + DICTIONARY_2: int = 238 + DICTIONARY_3: int = 239 + INTEROP_JID: int = 245 + FB_JID: int = 246 + AD_JID: int = 247 + LIST_8: int = 248 + LIST_16: int = 249 + JID_PAIR: int = 250 + HEX_8: int = 251 + BINARY_8: int = 252 + BINARY_20: int = 253 + BINARY_32: int = 254 + NIBBLE_8: int = 255 + PACKED_MAX: int = 127 + + +TAGS = BinaryTags() + + +@dataclass(slots=True) +class BinaryNode: + tag: str + attrs: dict[str, str] = field(default_factory=dict) + content: bytes | str | list["BinaryNode"] | None = None + + def child(self, tag: str) -> "BinaryNode | None": + if not isinstance(self.content, list): + return None + return next((item for item in self.content if item.tag == tag), None) + + def children(self, tag: str | None = None) -> list["BinaryNode"]: + if not isinstance(self.content, list): + return [] + return [item for item in self.content if tag is None or item.tag == tag] + + +class TokenTable: + """Versioned WABinary token lookup. + + Production can inject token tables generated from a pinned WhatsApp Web protocol snapshot. + Encoding always has a raw-string fallback, while decoding rejects unknown token references + instead of silently corrupting nodes. + """ + + def __init__(self, single: Sequence[str | None] = (), double: Sequence[Sequence[str | None]] = (), *, version: int | None = None): + self.single = tuple(single) + self.double = tuple(tuple(row) for row in double) + self.version = version + self._map: dict[str, tuple[int | None, int]] = {} + for idx, value in enumerate(self.single): + if value: + self._map[str(value)] = (None, idx) + for d, row in enumerate(self.double): + for idx, value in enumerate(row): + if value: + self._map[str(value)] = (d, idx) + + def token_for(self, value: str) -> tuple[int | None, int] | None: + return self._map.get(value) + + def decode_single(self, idx: int) -> str: + if not (0 <= idx < len(self.single)) or self.single[idx] is None: + raise BinaryNodeError(f"Unknown WABinary single-byte token {idx}") + return str(self.single[idx]) + + def decode_double(self, dictionary: int, idx: int) -> str: + if not (0 <= dictionary < len(self.double)) or not (0 <= idx < len(self.double[dictionary])) or self.double[dictionary][idx] is None: + raise BinaryNodeError(f"Unknown WABinary double-byte token {dictionary}:{idx}") + return str(self.double[dictionary][idx]) + + +EMPTY_TOKENS = TokenTable() + + +def _is_nibble(value: str) -> bool: + return bool(value) and len(value) <= TAGS.PACKED_MAX and all(ch.isdigit() or ch in "-." for ch in value) + + +def _is_hex(value: str) -> bool: + return bool(value) and len(value) <= TAGS.PACKED_MAX and all(ch in "0123456789ABCDEF" for ch in value) + + +def _pack(value: str, *, hex_mode: bool) -> bytes: + def cv(ch: str) -> int: + if ch == "\0": return 15 + if not hex_mode: + if ch == "-": return 10 + if ch == ".": return 11 + if ch.isdigit(): return ord(ch) - 48 + else: + if ch.isdigit(): return ord(ch) - 48 + if "A" <= ch <= "F": return 10 + ord(ch) - ord("A") + raise BinaryNodeError(f"Cannot pack character {ch!r}") + out = bytearray([TAGS.HEX_8 if hex_mode else TAGS.NIBBLE_8]) + size = (len(value) + 1) // 2 + out.append(size | (0x80 if len(value) % 2 else 0)) + for i in range(0, len(value), 2): + a = cv(value[i]) + b = cv(value[i + 1] if i + 1 < len(value) else "\0") + out.append((a << 4) | b) + return bytes(out) + + +def _unpack(tag: int, raw: bytes, pos: int) -> tuple[str, int]: + if pos >= len(raw): raise BinaryNodeError("Truncated packed string") + size_byte = raw[pos]; pos += 1 + nbytes = size_byte & 0x7F + odd = bool(size_byte & 0x80) + if pos + nbytes > len(raw): raise BinaryNodeError("Truncated packed string") + chars: list[str] = [] + for byte in raw[pos:pos+nbytes]: + for v in ((byte >> 4) & 15, byte & 15): + if tag == TAGS.HEX_8: + if v < 10: chars.append(chr(48 + v)) + elif v < 16: chars.append(chr(ord("A") + v - 10) if v < 15 else "\0") + else: raise BinaryNodeError("Invalid hex nibble") + else: + if v <= 9: chars.append(chr(48 + v)) + elif v == 10: chars.append("-") + elif v == 11: chars.append(".") + elif v == 15: chars.append("\0") + else: raise BinaryNodeError("Invalid numeric nibble") + if odd: + chars = chars[:-1] + return "".join(chars).replace("\0", ""), pos + nbytes + + +class BinaryNodeCodec: + def __init__(self, tokens: TokenTable = EMPTY_TOKENS, *, tags: BinaryTags = TAGS): + self.tokens = tokens + self.tags = tags + + def encode(self, node: BinaryNode, *, compressed: bool = False) -> bytes: + body = bytearray() + self._write_node(body, node) + if compressed: + return bytes([2]) + zlib.compress(bytes(body)) + return bytes([0]) + bytes(body) + + def decode(self, payload: bytes) -> BinaryNode: + raw = bytes(payload) + if not raw: + raise BinaryNodeError("Empty WABinary payload") + flag, body = raw[0], raw[1:] + if flag & 2: + try: + body = zlib.decompress(body) + except zlib.error as exc: + raise BinaryNodeError("Invalid compressed WABinary payload") from exc + node, pos = self._read_node(body, 0) + if pos != len(body): + raise BinaryNodeError(f"Trailing WABinary bytes: {len(body)-pos}") + return node + + def _list_start(self, out: bytearray, size: int) -> None: + if size == 0: out.append(self.tags.LIST_EMPTY) + elif size < 256: out.extend((self.tags.LIST_8, size)) + elif size < 65536: out.append(self.tags.LIST_16); out.extend(size.to_bytes(2, "big")) + else: raise BinaryNodeError("WABinary list too large") + + def _write_len(self, out: bytearray, length: int) -> None: + if length < 256: out.extend((self.tags.BINARY_8, length)) + elif length < (1 << 20): out.append(self.tags.BINARY_20); out.extend(((length >> 16) & 15, (length >> 8) & 255, length & 255)) + elif length < (1 << 32): out.append(self.tags.BINARY_32); out.extend(length.to_bytes(4, "big")) + else: raise BinaryNodeError("WABinary value too large") + + def _write_raw(self, out: bytearray, raw: bytes) -> None: + self._write_len(out, len(raw)); out.extend(raw) + + def _write_jid(self, out: bytearray, jid: JID) -> None: + if jid.device is not None: + out.extend((self.tags.AD_JID, jid.domain_type & 255, jid.device & 255)) + self._write_string(out, jid.user) + else: + out.append(self.tags.JID_PAIR) + if jid.user: self._write_string(out, jid.user) + else: out.append(self.tags.LIST_EMPTY) + self._write_string(out, jid.server) + + def _write_string(self, out: bytearray, value: str | None) -> None: + if value is None: + out.append(self.tags.LIST_EMPTY); return + token = self.tokens.token_for(value) + if token is not None: + dictionary, idx = token + if dictionary is not None: + if not 0 <= dictionary <= 3: raise BinaryNodeError("Token dictionary index out of range") + out.append(self.tags.DICTIONARY_0 + dictionary) + out.append(idx & 255); return + if _is_nibble(value): out.extend(_pack(value, hex_mode=False)); return + if _is_hex(value): out.extend(_pack(value, hex_mode=True)); return + try: + jid = parse_jid(value) + except ValueError: + jid = None + if jid is not None: + self._write_jid(out, jid); return + self._write_raw(out, value.encode("utf-8")) + + def _write_node(self, out: bytearray, node: BinaryNode) -> None: + if not node.tag: raise BinaryNodeError("Node tag cannot be empty") + attrs = [(str(k), str(v)) for k, v in (node.attrs or {}).items() if v is not None] + has_content = node.content is not None + self._list_start(out, 1 + 2 * len(attrs) + (1 if has_content else 0)) + self._write_string(out, node.tag) + for key, value in attrs: + self._write_string(out, key); self._write_string(out, value) + content = node.content + if content is None: return + if isinstance(content, str): self._write_string(out, content) + elif isinstance(content, (bytes, bytearray, memoryview)): self._write_raw(out, bytes(content)) + elif isinstance(content, list): + self._list_start(out, len(content)) + for child in content: + if not isinstance(child, BinaryNode): raise BinaryNodeError("Node child must be BinaryNode") + self._write_node(out, child) + else: raise BinaryNodeError(f"Unsupported node content: {type(content).__name__}") + + def _read_n(self, raw: bytes, pos: int, n: int) -> tuple[bytes, int]: + if pos + n > len(raw): raise BinaryNodeError("WABinary end of stream") + return raw[pos:pos+n], pos+n + + def _read_list_size(self, raw: bytes, pos: int, tag: int | None = None) -> tuple[int, int]: + if tag is None: + b, pos = self._read_n(raw, pos, 1); tag = b[0] + if tag == self.tags.LIST_EMPTY: return 0, pos + if tag == self.tags.LIST_8: + b, pos = self._read_n(raw, pos, 1); return b[0], pos + if tag == self.tags.LIST_16: + b, pos = self._read_n(raw, pos, 2); return int.from_bytes(b, "big"), pos + raise BinaryNodeError(f"Invalid WABinary list tag {tag}") + + def _read_len(self, raw: bytes, pos: int, tag: int) -> tuple[int, int]: + if tag == self.tags.BINARY_8: + b, pos = self._read_n(raw, pos, 1); return b[0], pos + if tag == self.tags.BINARY_20: + b, pos = self._read_n(raw, pos, 3); return ((b[0] & 15) << 16) | (b[1] << 8) | b[2], pos + if tag == self.tags.BINARY_32: + b, pos = self._read_n(raw, pos, 4); return int.from_bytes(b, "big"), pos + raise BinaryNodeError(f"Not a binary-length tag: {tag}") + + def _read_string(self, raw: bytes, pos: int, first: int | None = None) -> tuple[str, int]: + if first is None: + b, pos = self._read_n(raw, pos, 1); first = b[0] + if 1 <= first < len(self.tokens.single): return self.tokens.decode_single(first), pos + if self.tags.DICTIONARY_0 <= first <= self.tags.DICTIONARY_3: + b, pos = self._read_n(raw, pos, 1); return self.tokens.decode_double(first - self.tags.DICTIONARY_0, b[0]), pos + if first == self.tags.LIST_EMPTY: return "", pos + if first in (self.tags.BINARY_8, self.tags.BINARY_20, self.tags.BINARY_32): + length, pos = self._read_len(raw, pos, first); b, pos = self._read_n(raw, pos, length) + try: return b.decode("utf-8"), pos + except UnicodeDecodeError as exc: raise BinaryNodeError("Binary string is not UTF-8") from exc + if first in (self.tags.NIBBLE_8, self.tags.HEX_8): return _unpack(first, raw, pos) + if first == self.tags.JID_PAIR: + user, pos = self._read_string(raw, pos); server, pos = self._read_string(raw, pos) + if not server: raise BinaryNodeError("JID pair has empty server") + return f"{user}@{server}", pos + if first == self.tags.AD_JID: + b, pos = self._read_n(raw, pos, 2); domain_type, device = b[0], b[1] + user, pos = self._read_string(raw, pos) + server = {1:"lid",128:"hosted",129:"hosted.lid"}.get(domain_type, "s.whatsapp.net") + return str(JID(user, server, device=device, domain_type=domain_type)), pos + if first == self.tags.FB_JID: + user, pos = self._read_string(raw, pos) + b, pos = self._read_n(raw, pos, 2); device = int.from_bytes(b, "big") + server, pos = self._read_string(raw, pos) + if not server: raise BinaryNodeError("FB JID has empty server") + return f"{user}:{device}@{server}", pos + if first == self.tags.INTEROP_JID: + user, pos = self._read_string(raw, pos) + b, pos = self._read_n(raw, pos, 2); device = int.from_bytes(b, "big") + b, pos = self._read_n(raw, pos, 2); integrator = int.from_bytes(b, "big") + # Current wire format may omit the optional server; in that case preserve + # the protocol-default interop domain instead of consuming the next field. + before = pos + try: + server, pos = self._read_string(raw, pos) + except BinaryNodeError: + server, pos = "interop", before + if not server: server = "interop" + return f"{integrator}-{user}:{device}@{server}", pos + raise BinaryNodeError(f"Unknown WABinary string token/tag {first}") + + def _read_content(self, raw: bytes, pos: int, first: int) -> tuple[bytes | str | list[BinaryNode], int]: + if first in (self.tags.LIST_EMPTY, self.tags.LIST_8, self.tags.LIST_16): + size, pos = self._read_list_size(raw, pos, first) + items: list[BinaryNode] = [] + for _ in range(size): + node, pos = self._read_node(raw, pos); items.append(node) + return items, pos + if first in (self.tags.BINARY_8, self.tags.BINARY_20, self.tags.BINARY_32): + length, pos = self._read_len(raw, pos, first) + return self._read_n(raw, pos, length) + return self._read_string(raw, pos, first) + + def _read_node(self, raw: bytes, pos: int) -> tuple[BinaryNode, int]: + size, pos = self._read_list_size(raw, pos) + if size <= 0: raise BinaryNodeError("Invalid empty WABinary node") + tag, pos = self._read_string(raw, pos) + if not tag: raise BinaryNodeError("WABinary node tag is empty") + attrs: dict[str, str] = {} + for _ in range((size - 1) // 2): + key, pos = self._read_string(raw, pos); value, pos = self._read_string(raw, pos); attrs[key] = value + content = None + if size % 2 == 0: + b, pos = self._read_n(raw, pos, 1); content, pos = self._read_content(raw, pos, b[0]) + return BinaryNode(tag=tag, attrs=attrs, content=content), pos diff --git a/src/postmaster/whatsapp_v990/cert.py b/src/postmaster/whatsapp_v990/cert.py new file mode 100644 index 00000000..ff87b79f --- /dev/null +++ b/src/postmaster/whatsapp_v990/cert.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +"""Clean-room WhatsApp Noise certificate-chain parsing and verification.""" + +from dataclasses import dataclass +import time + +from .crypto import WA_CERT_PUBLIC_KEY, xeddsa_verify +from .proto import ProtoError, decode_fields + + +class NoiseCertificateError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class NoiseCertificateDetails: + serial: int | None + issuer_serial: int | None + key: bytes | None + not_before: int | None + not_after: int | None + + +@dataclass(frozen=True, slots=True) +class VerifiedNoiseCertificateChain: + leaf: NoiseCertificateDetails + intermediate: NoiseCertificateDetails + chain_signatures_verified: bool = True + root_serial: int = 0 + + +def _field_bytes(raw: bytes, number: int, *, required: bool = True) -> bytes | None: + try: + fields = decode_fields(bytes(raw)) + except ProtoError as exc: + raise NoiseCertificateError("Invalid Noise certificate protobuf") from exc + values = [field.value for field in fields if field.number == number and isinstance(field.value, bytes)] + if not values: + if required: + raise NoiseCertificateError(f"Missing Noise certificate bytes field {number}") + return None + return bytes(values[-1]) + + +def _field_int(raw: bytes, number: int) -> int | None: + try: + fields = decode_fields(bytes(raw)) + except ProtoError as exc: + raise NoiseCertificateError("Invalid Noise certificate details protobuf") from exc + values = [field.value for field in fields if field.number == number and isinstance(field.value, int)] + return int(values[-1]) if values else None + + +def _certificate(raw: bytes) -> tuple[bytes, bytes]: + details = _field_bytes(raw, 1) + signature = _field_bytes(raw, 2) + assert details is not None and signature is not None + if len(signature) != 64: + raise NoiseCertificateError("Noise certificate signature must be 64 bytes") + return details, signature + + +def decode_noise_certificate_details(raw: bytes) -> NoiseCertificateDetails: + key = _field_bytes(raw, 3, required=False) + if key is not None and len(key) != 32: + raise NoiseCertificateError("Noise certificate Curve25519 key must be 32 bytes") + return NoiseCertificateDetails( + serial=_field_int(raw, 1), + issuer_serial=_field_int(raw, 2), + key=key, + not_before=_field_int(raw, 4), + not_after=_field_int(raw, 5), + ) + + +def _check_validity(details: NoiseCertificateDetails, *, now: int) -> None: + if details.not_before is not None and now < details.not_before: + raise NoiseCertificateError("Noise certificate is not valid yet") + if details.not_after is not None and now > details.not_after: + raise NoiseCertificateError("Noise certificate is expired") + + +def verify_noise_certificate_chain( + raw: bytes, + *, + root_public_key: bytes = WA_CERT_PUBLIC_KEY, + root_serial: int = 0, + now: int | None = None, +) -> VerifiedNoiseCertificateChain: + """Verify WhatsApp's leaf -> intermediate -> pinned root Noise certificate chain. + + The root key/serial defaults are the public WhatsApp long-term Noise trust anchor. + Callers may inject another root only for deterministic unit tests. + """ + leaf_raw = _field_bytes(raw, 1) + intermediate_raw = _field_bytes(raw, 2) + assert leaf_raw is not None and intermediate_raw is not None + + leaf_details_raw, leaf_signature = _certificate(leaf_raw) + intermediate_details_raw, intermediate_signature = _certificate(intermediate_raw) + + leaf = decode_noise_certificate_details(leaf_details_raw) + intermediate = decode_noise_certificate_details(intermediate_details_raw) + if intermediate.issuer_serial != int(root_serial): + raise NoiseCertificateError( + f"Noise intermediate issuer serial {intermediate.issuer_serial!r} does not match pinned root {root_serial}" + ) + if intermediate.key is None: + raise NoiseCertificateError("Noise intermediate certificate has no public key") + + if not xeddsa_verify(root_public_key, intermediate_details_raw, intermediate_signature): + raise NoiseCertificateError("Noise intermediate certificate signature verification failed") + if not xeddsa_verify(intermediate.key, leaf_details_raw, leaf_signature): + raise NoiseCertificateError("Noise leaf certificate signature verification failed") + + current = int(time.time()) if now is None else int(now) + _check_validity(intermediate, now=current) + _check_validity(leaf, now=current) + return VerifiedNoiseCertificateChain( + leaf=leaf, + intermediate=intermediate, + chain_signatures_verified=True, + root_serial=int(root_serial), + ) + + +__all__ = [ + "NoiseCertificateError", + "NoiseCertificateDetails", + "VerifiedNoiseCertificateChain", + "decode_noise_certificate_details", + "verify_noise_certificate_chain", +] diff --git a/src/postmaster/whatsapp_v990/client_payload.py b/src/postmaster/whatsapp_v990/client_payload.py new file mode 100644 index 00000000..9dc4324c --- /dev/null +++ b/src/postmaster/whatsapp_v990/client_payload.py @@ -0,0 +1,239 @@ +from __future__ import annotations + +"""Minimal clean-room protobuf builders for WhatsApp Web ClientPayload. + +Field numbers/enums are pinned from the current public WhatsApp Web proto schema. The builders +cover only the registration/login subset Postmaster needs and deliberately preserve the generic +protobuf implementation in proto.py instead of importing generated third-party code. +""" + +from dataclasses import dataclass +import hashlib +from typing import Iterable + +from .proto import field_bytes, field_message, field_varint +from .signal_keys import SignedPreKey + +CURRENT_WA_WEB_VERSION = (2, 3000, 1048032155) +DEVICE_PROPS_VERSION = (10, 15, 7) + +# ClientPayload.UserAgent enums. +USER_AGENT_PLATFORM_WEB = 14 +USER_AGENT_RELEASE_CHANNEL_RELEASE = 0 +WEB_SUBPLATFORM_WEB_BROWSER = 0 + +# ClientPayload enums. +CONNECT_TYPE_WIFI_UNKNOWN = 1 +CONNECT_REASON_USER_ACTIVATED = 1 + +# DeviceProps enums. +DEVICE_PLATFORM_CHROME = 1 + +# Signal/WhatsApp public-key bundle type. +KEY_BUNDLE_TYPE = b"\x05" + + +class ClientPayloadError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class RegistrationKeys: + registration_id: int + identity_public_key: bytes + signed_pre_key: SignedPreKey + + +def parse_web_version(value: str | Iterable[int] | None) -> tuple[int, int, int]: + if value is None: + return CURRENT_WA_WEB_VERSION + if isinstance(value, str): + clean = value.strip().split("-", 1)[0] + parts = clean.split(".") + else: + parts = [str(x) for x in value] + if len(parts) != 3: + raise ClientPayloadError("WhatsApp Web version must contain exactly three numeric components") + try: + parsed = tuple(int(x) for x in parts) + except ValueError as exc: + raise ClientPayloadError("WhatsApp Web version components must be numeric") from exc + if any(x < 0 or x > 0xFFFFFFFF for x in parsed): + raise ClientPayloadError("WhatsApp Web version component is outside uint32 range") + return parsed # type: ignore[return-value] + + +def encode_big_endian(value: int, width: int | None = None) -> bytes: + value = int(value) + if value < 0: + raise ClientPayloadError("Big-endian value must be non-negative") + if width is None: + width = max(1, (value.bit_length() + 7) // 8) + if width <= 0 or value >= (1 << (8 * width)): + raise ClientPayloadError("Big-endian value does not fit requested width") + return value.to_bytes(width, "big") + + +def encode_app_version(version: tuple[int, int, int]) -> bytes: + primary, secondary, tertiary = version + return b"".join((field_varint(1, primary), field_varint(2, secondary), field_varint(3, tertiary))) + + +def encode_user_agent( + version: tuple[int, int, int], + *, + locale_language: str = "en", + locale_country: str = "US", + device: str = "Desktop", + os_version: str = "0.1", + os_build_number: str = "0.1", +) -> bytes: + return b"".join( + ( + field_varint(1, USER_AGENT_PLATFORM_WEB), + field_message(2, [encode_app_version(version)]), + field_bytes(3, "000"), + field_bytes(4, "000"), + field_bytes(5, os_version), + field_bytes(7, device), + field_bytes(8, os_build_number), + field_varint(10, USER_AGENT_RELEASE_CHANNEL_RELEASE), + field_bytes(11, locale_language), + field_bytes(12, locale_country), + ) + ) + + +def encode_web_info() -> bytes: + return field_varint(4, WEB_SUBPLATFORM_WEB_BROWSER) + + +def encode_history_sync_config() -> bytes: + # Mirrors the currently published desktop registration capability subset. Optional fields + # intentionally stay omitted rather than being serialized with guessed/default values. + return b"".join( + ( + field_varint(3, 10240), # storageQuotaMb + field_varint(4, 1), # inlineInitialPayloadInE2EeMsg + field_varint(6, 0), # supportCallLogHistory + field_varint(7, 1), # supportBotUserAgentChatHistory + field_varint(8, 1), # supportCagReactionsAndPolls + field_varint(9, 1), # supportBizHostedMsg + field_varint(10, 1), # supportRecentSyncChunkMessageCountTuning + field_varint(11, 1), # supportHostedGroupMsg + field_varint(12, 1), # supportFbidBotChatHistory + field_varint(14, 1), # supportMessageAssociation + field_varint(15, 0), # supportGroupHistory + ) + ) + + +def encode_device_props( + *, + os_name: str = "Linux", + platform_type: int = DEVICE_PLATFORM_CHROME, + require_full_sync: bool = False, +) -> bytes: + return b"".join( + ( + field_bytes(1, os_name), + field_message(2, [encode_app_version(DEVICE_PROPS_VERSION)]), + field_varint(3, int(platform_type)), + field_varint(4, 1 if require_full_sync else 0), + field_message(5, [encode_history_sync_config()]), + ) + ) + + +def build_registration_payload( + keys: RegistrationKeys, + *, + version: tuple[int, int, int] | str | None = None, + os_name: str = "Linux", + locale_country: str = "US", + require_full_sync: bool = False, +) -> bytes: + resolved = parse_web_version(version) + identity = bytes(keys.identity_public_key) + signed_pair = keys.signed_pre_key.key_pair + if len(identity) != 32 or len(signed_pair.public) != 32: + raise ClientPayloadError("WhatsApp registration identity/pre-key public keys must be 32 bytes") + if len(keys.signed_pre_key.signature) != 64: + raise ClientPayloadError("WhatsApp signed-pre-key signature must be 64 bytes") + if not 0 <= int(keys.registration_id) <= 0x3FFF: + raise ClientPayloadError("WhatsApp registration id must fit 14 bits") + + version_text = ".".join(str(x) for x in resolved).encode("ascii") + build_hash = hashlib.md5(version_text, usedforsecurity=False).digest() + pairing = b"".join( + ( + field_bytes(1, encode_big_endian(keys.registration_id, 4)), + field_bytes(2, KEY_BUNDLE_TYPE), + field_bytes(3, identity), + field_bytes(4, encode_big_endian(keys.signed_pre_key.key_id, 3)), + field_bytes(5, signed_pair.public), + field_bytes(6, keys.signed_pre_key.signature), + field_bytes(7, build_hash), + field_bytes( + 8, + encode_device_props( + os_name=os_name, + platform_type=DEVICE_PLATFORM_CHROME, + require_full_sync=require_full_sync, + ), + ), + ) + ) + return b"".join( + ( + field_varint(3, 0), # passive=false + field_message(5, [encode_user_agent(resolved, locale_country=locale_country)]), + field_message(6, [encode_web_info()]), + field_varint(12, CONNECT_TYPE_WIFI_UNKNOWN), + field_varint(13, CONNECT_REASON_USER_ACTIVATED), + field_message(19, [pairing]), + field_varint(33, 0), # pull=false + ) + ) + + +def build_login_payload( + *, + username: int, + device: int, + version: tuple[int, int, int] | str | None = None, + locale_country: str = "US", +) -> bytes: + resolved = parse_web_version(version) + if username < 0 or device < 0: + raise ClientPayloadError("WhatsApp login username/device must be non-negative") + return b"".join( + ( + field_varint(1, int(username)), + field_varint(3, 1), # passive=true + field_message(5, [encode_user_agent(resolved, locale_country=locale_country)]), + field_message(6, [encode_web_info()]), + field_varint(12, CONNECT_TYPE_WIFI_UNKNOWN), + field_varint(13, CONNECT_REASON_USER_ACTIVATED), + field_varint(18, int(device)), + field_varint(33, 1), # pull=true + field_varint(41, 0), # lidDbMigrated=false + ) + ) + + +__all__ = [ + "CURRENT_WA_WEB_VERSION", + "DEVICE_PROPS_VERSION", + "RegistrationKeys", + "ClientPayloadError", + "parse_web_version", + "encode_big_endian", + "encode_app_version", + "encode_user_agent", + "encode_web_info", + "encode_history_sync_config", + "encode_device_props", + "build_registration_payload", + "build_login_payload", +] diff --git a/src/postmaster/whatsapp_v990/crypto.py b/src/postmaster/whatsapp_v990/crypto.py new file mode 100644 index 00000000..e80f8893 --- /dev/null +++ b/src/postmaster/whatsapp_v990/crypto.py @@ -0,0 +1,416 @@ +from __future__ import annotations + +"""Clean-room WhatsApp/Signal cryptographic primitives for Postmaster v9.9. + +Only public protocol specifications are used here. The module intentionally keeps the +WhatsApp-specific policy thin: Curve25519/XEdDSA, Noise XX helpers, and media key derivation. +Long-lived secrets are never logged or returned by the public MCP surface. +""" + +from dataclasses import dataclass +import hashlib +import hmac +import os +from typing import Final + +from cryptography.hazmat.primitives import hashes, padding, serialization +from cryptography.hazmat.primitives.asymmetric import x25519 +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.kdf.hkdf import HKDF + +P: Final[int] = 2**255 - 19 +Q: Final[int] = 2**252 + 27742317777372353535851937790883648493 +D: Final[int] = (-121665 * pow(121666, P - 2, P)) % P +SQRT_M1: Final[int] = pow(2, (P - 1) // 4, P) +NOISE_MODE: Final[bytes] = b"Noise_XX_25519_AESGCM_SHA256\x00\x00\x00\x00" +NOISE_WA_HEADER: Final[bytes] = bytes((87, 65, 6, 3)) +WA_CERT_PUBLIC_KEY: Final[bytes] = bytes.fromhex("142375574d0a587166aae71ebe516437c4a28b73e3695c6ce1f7f9545da8ee6b") + + +class CryptoError(ValueError): + pass + + +@dataclass(frozen=True) +class CurveKeyPair: + private: bytes + public: bytes + + +def clamp_curve25519_scalar(raw: bytes) -> bytes: + if len(raw) != 32: + raise CryptoError("Curve25519 private key must be 32 bytes") + b = bytearray(raw) + b[0] &= 248 + b[31] &= 127 + b[31] |= 64 + return bytes(b) + + +def generate_curve_keypair() -> CurveKeyPair: + private = x25519.X25519PrivateKey.generate() + priv = private.private_bytes(serialization.Encoding.Raw, serialization.PrivateFormat.Raw, serialization.NoEncryption()) + pub = private.public_key().public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) + return CurveKeyPair(private=priv, public=pub) + + +def curve_public_from_private(private: bytes) -> bytes: + key = x25519.X25519PrivateKey.from_private_bytes(private) + return key.public_key().public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) + + +def curve_shared_key(private: bytes, public: bytes) -> bytes: + if len(private) != 32 or len(public) != 32: + raise CryptoError("Curve25519 keys must be 32 bytes") + return x25519.X25519PrivateKey.from_private_bytes(private).exchange(x25519.X25519PublicKey.from_public_bytes(public)) + + +def signal_public_key(public: bytes) -> bytes: + if len(public) == 33 and public[0] == 5: + return public + if len(public) != 32: + raise CryptoError("Signal Curve25519 public key must be 32 bytes") + return b"\x05" + public + + +Point = tuple[int, int] +IDENTITY: Final[Point] = (0, 1) + + +def _inv(x: int) -> int: + return pow(x % P, P - 2, P) if x % P else 0 + + +def _sqrt(v: int) -> int | None: + v %= P + x = pow(v, (P + 3) // 8, P) + if (x * x - v) % P: + x = (x * SQRT_M1) % P + if (x * x - v) % P: + return None + return x + + +def _point_from_y(y: int, sign: int) -> Point: + if not 0 <= y < P: + raise CryptoError("Invalid Edwards y coordinate") + y2 = y * y % P + x2 = (y2 - 1) * _inv(1 + D * y2) % P + x = _sqrt(x2) + if x is None: + raise CryptoError("Point is not on Curve25519 Edwards form") + if (x & 1) != (sign & 1): + x = (-x) % P + point = (x, y) + if not _on_curve(point): + raise CryptoError("Decoded point is not on curve") + return point + + +def _on_curve(point: Point) -> bool: + x, y = point + return (-x * x + y * y - 1 - D * x * x * y * y) % P == 0 + + +def _point_add(a: Point, b: Point) -> Point: + x1, y1 = a + x2, y2 = b + t = D * x1 * x2 * y1 * y2 % P + den_x = _inv(1 + t) + den_y = _inv(1 - t) + x3 = (x1 * y2 + y1 * x2) * den_x % P + y3 = (y1 * y2 + x1 * x2) * den_y % P + return x3, y3 + + +def _point_neg(point: Point) -> Point: + return (-point[0]) % P, point[1] + + +def _scalar_mult(k: int, point: Point) -> Point: + k %= Q + acc = IDENTITY + cur = point + while k: + if k & 1: + acc = _point_add(acc, cur) + cur = _point_add(cur, cur) + k >>= 1 + return acc + + +def _encode_point(point: Point) -> bytes: + x, y = point + raw = bytearray(int(y).to_bytes(32, "little")) + raw[31] = (raw[31] & 0x7F) | ((x & 1) << 7) + return bytes(raw) + + +def _decode_point(raw: bytes) -> Point: + if len(raw) != 32: + raise CryptoError("Compressed Edwards point must be 32 bytes") + value = int.from_bytes(raw, "little") + sign = value >> 255 + y = value & ((1 << 255) - 1) + return _point_from_y(y, sign) + + +def _convert_montgomery_u(public: bytes) -> Point: + if len(public) == 33 and public[0] == 5: + public = public[1:] + if len(public) != 32: + raise CryptoError("Montgomery public key must be 32 bytes") + u = int.from_bytes(public, "little") & ((1 << 255) - 1) + y = (u - 1) * _inv(u + 1) % P + return _point_from_y(y, 0) + + +BASE_POINT: Final[Point] = _convert_montgomery_u((9).to_bytes(32, "little")) + + +def _xeddsa_keypair(private: bytes) -> tuple[Point, int]: + scalar_bytes = clamp_curve25519_scalar(private) + k = int.from_bytes(scalar_bytes, "little") % Q + e = _scalar_mult(k, BASE_POINT) + a = (-k) % Q if (e[0] & 1) else k + A = (e[0] if not (e[0] & 1) else (-e[0]) % P, e[1]) + if A[0] & 1: + raise CryptoError("Failed to normalize XEdDSA public key") + return A, a + + +def _hash_int(data: bytes) -> int: + return int.from_bytes(hashlib.sha512(data).digest(), "little") + + +def _hash_i(index: int, data: bytes) -> int: + if not 0 <= index < 256: + raise CryptoError("XEdDSA domain index must fit one byte") + prefix = ((1 << 256) - 1 - index).to_bytes(32, "little") + return _hash_int(prefix + data) + + +def xeddsa_sign(private: bytes, message: bytes, *, random64: bytes | None = None) -> bytes: + if len(private) != 32: + raise CryptoError("XEdDSA private key must be 32 bytes") + z = random64 if random64 is not None else os.urandom(64) + if len(z) != 64: + raise CryptoError("XEdDSA randomizer must be 64 bytes") + A, a = _xeddsa_keypair(private) + a_bytes = a.to_bytes(32, "little") + r = _hash_i(1, a_bytes + bytes(message) + z) % Q + R = _scalar_mult(r, BASE_POINT) + Rb = _encode_point(R) + Ab = _encode_point(A) + h = _hash_int(Rb + Ab + bytes(message)) % Q + s = (r + h * a) % Q + return Rb + s.to_bytes(32, "little") + + +def xeddsa_public_edwards(public: bytes) -> bytes: + return _encode_point(_convert_montgomery_u(public)) + + +def xeddsa_verify(public: bytes, message: bytes, signature: bytes) -> bool: + try: + if len(signature) != 64: + return False + A = _convert_montgomery_u(public) + Rb = signature[:32] + R = _decode_point(Rb) + s = int.from_bytes(signature[32:], "little") + if s >= Q: + return False + Ab = _encode_point(A) + h = _hash_int(Rb + Ab + bytes(message)) % Q + check = _point_add(_scalar_mult(s, BASE_POINT), _point_neg(_scalar_mult(h, A))) + return hmac.compare_digest(_encode_point(check), Rb) + except (CryptoError, ValueError): + return False + + +def hkdf_sha256(ikm: bytes, length: int, *, salt: bytes | None = None, info: bytes = b"") -> bytes: + return HKDF(algorithm=hashes.SHA256(), length=int(length), salt=salt, info=info).derive(bytes(ikm)) + + +def aes_cbc_encrypt(key: bytes, iv: bytes, plaintext: bytes) -> bytes: + padder = padding.PKCS7(128).padder() + padded = padder.update(bytes(plaintext)) + padder.finalize() + enc = Cipher(algorithms.AES(bytes(key)), modes.CBC(bytes(iv))).encryptor() + return enc.update(padded) + enc.finalize() + + +def aes_cbc_decrypt(key: bytes, iv: bytes, ciphertext: bytes) -> bytes: + dec = Cipher(algorithms.AES(bytes(key)), modes.CBC(bytes(iv))).decryptor() + padded = dec.update(bytes(ciphertext)) + dec.finalize() + unpadder = padding.PKCS7(128).unpadder() + return unpadder.update(padded) + unpadder.finalize() + + +_MEDIA_INFO = { + "image": b"WhatsApp Image Keys", + "video": b"WhatsApp Video Keys", + "audio": b"WhatsApp Audio Keys", + "document": b"WhatsApp Document Keys", + "sticker": b"WhatsApp Image Keys", +} + + +def encrypt_media(plaintext: bytes, media_type: str, *, media_key: bytes | None = None) -> dict[str, bytes]: + kind = str(media_type).lower() + if kind not in _MEDIA_INFO: + raise CryptoError(f"Unsupported WhatsApp media type: {media_type}") + key = media_key or os.urandom(32) + if len(key) != 32: + raise CryptoError("WhatsApp media key must be 32 bytes") + expanded = hkdf_sha256(key, 112, salt=None, info=_MEDIA_INFO[kind]) + iv, cipher_key, mac_key = expanded[:16], expanded[16:48], expanded[48:80] + encrypted = aes_cbc_encrypt(cipher_key, iv, bytes(plaintext)) + mac = hmac.new(mac_key, iv + encrypted, hashlib.sha256).digest()[:10] + blob = encrypted + mac + return { + "media_key": key, + "file_enc_sha256": hashlib.sha256(blob).digest(), + "file_sha256": hashlib.sha256(bytes(plaintext)).digest(), + "encrypted": blob, + } + + +def decrypt_media(blob: bytes, media_type: str, media_key: bytes, *, expected_file_sha256: bytes | None = None) -> bytes: + kind = str(media_type).lower() + if kind not in _MEDIA_INFO: + raise CryptoError(f"Unsupported WhatsApp media type: {media_type}") + if len(blob) < 11 or len(media_key) != 32: + raise CryptoError("Invalid WhatsApp media payload") + expanded = hkdf_sha256(media_key, 112, salt=None, info=_MEDIA_INFO[kind]) + iv, cipher_key, mac_key = expanded[:16], expanded[16:48], expanded[48:80] + encrypted, mac = bytes(blob[:-10]), bytes(blob[-10:]) + expected = hmac.new(mac_key, iv + encrypted, hashlib.sha256).digest()[:10] + if not hmac.compare_digest(mac, expected): + raise CryptoError("WhatsApp media MAC mismatch") + plaintext = aes_cbc_decrypt(cipher_key, iv, encrypted) + if expected_file_sha256 is not None and not hmac.compare_digest(hashlib.sha256(plaintext).digest(), expected_file_sha256): + raise CryptoError("WhatsApp media SHA-256 mismatch") + return plaintext + + +def _noise_iv(counter: int) -> bytes: + if counter < 0 or counter > 0xFFFFFFFF: + raise CryptoError("Noise nonce counter exhausted") + return b"\x00" * 8 + int(counter).to_bytes(4, "big") + + +@dataclass +class NoiseTransport: + write_key: bytes + read_key: bytes + write_counter: int = 0 + read_counter: int = 0 + + def encrypt(self, plaintext: bytes) -> bytes: + iv = _noise_iv(self.write_counter) + self.write_counter += 1 + return AESGCM(self.write_key).encrypt(iv, bytes(plaintext), b"") + + def decrypt(self, ciphertext: bytes) -> bytes: + iv = _noise_iv(self.read_counter) + self.read_counter += 1 + try: + return AESGCM(self.read_key).decrypt(iv, bytes(ciphertext), b"") + except Exception as exc: + raise CryptoError("Noise transport authentication failed") from exc + + +class WhatsAppNoiseXX: + """Initiator-side Noise_XX_25519_AESGCM_SHA256 state used by WhatsApp Web.""" + + def __init__(self, ephemeral: CurveKeyPair, *, header: bytes = NOISE_WA_HEADER): + self.ephemeral = ephemeral + initial = hashlib.sha256(NOISE_MODE).digest() if len(NOISE_MODE) != 32 else NOISE_MODE + self.hash = initial + self.salt = initial + self.key = initial + self.counter = 0 + self.transport: NoiseTransport | None = None + self.authenticate(header) + self.authenticate(ephemeral.public) + + def authenticate(self, data: bytes) -> None: + if self.transport is None: + self.hash = hashlib.sha256(self.hash + bytes(data)).digest() + + def _derive_pair(self, data: bytes) -> tuple[bytes, bytes]: + material = hkdf_sha256(bytes(data), 64, salt=self.salt, info=b"") + return material[:32], material[32:] + + def mix_key(self, data: bytes) -> None: + self.salt, self.key = self._derive_pair(data) + self.counter = 0 + + def encrypt(self, plaintext: bytes) -> bytes: + if self.transport is not None: + return self.transport.encrypt(plaintext) + iv = _noise_iv(self.counter) + self.counter += 1 + out = AESGCM(self.key).encrypt(iv, bytes(plaintext), self.hash) + self.authenticate(out) + return out + + def decrypt(self, ciphertext: bytes) -> bytes: + if self.transport is not None: + return self.transport.decrypt(ciphertext) + iv = _noise_iv(self.counter) + self.counter += 1 + try: + out = AESGCM(self.key).decrypt(iv, bytes(ciphertext), self.hash) + except Exception as exc: + raise CryptoError("Noise handshake authentication failed") from exc + self.authenticate(ciphertext) + return out + + def process_server_hello(self, *, server_ephemeral: bytes, encrypted_static: bytes, encrypted_payload: bytes, + noise_static: CurveKeyPair) -> tuple[bytes, bytes]: + self.authenticate(server_ephemeral) + self.mix_key(curve_shared_key(self.ephemeral.private, server_ephemeral)) + server_static = self.decrypt(encrypted_static) + if len(server_static) != 32: + raise CryptoError("Noise server static key must be 32 bytes") + self.mix_key(curve_shared_key(self.ephemeral.private, server_static)) + cert_chain = self.decrypt(encrypted_payload) + # The live server identity must chain to WhatsApp's pinned public Noise root before + # the client sends its static key or any ClientPayload. + try: + from .cert import verify_noise_certificate_chain + verify_noise_certificate_chain(cert_chain) + except Exception as exc: + raise CryptoError("Noise server certificate validation failed") from exc + encrypted_client_static = self.encrypt(noise_static.public) + self.mix_key(curve_shared_key(noise_static.private, server_ephemeral)) + return encrypted_client_static, cert_chain + + def finish(self) -> NoiseTransport: + write, read = self._derive_pair(b"") + self.transport = NoiseTransport(write_key=write, read_key=read) + return self.transport + + +def frame_noise_payload(payload: bytes, *, intro: bytes | None = None) -> bytes: + size = len(payload) + if size > 0xFFFFFF: + raise CryptoError("WhatsApp Noise frame exceeds 24-bit length") + return (bytes(intro or b"") + size.to_bytes(3, "big") + bytes(payload)) + + +def split_noise_frames(buffer: bytes) -> tuple[list[bytes], bytes]: + frames: list[bytes] = [] + pos = 0 + data = bytes(buffer) + while len(data) - pos >= 3: + size = int.from_bytes(data[pos:pos+3], "big") + if len(data) - pos - 3 < size: + break + pos += 3 + frames.append(data[pos:pos+size]) + pos += size + return frames, data[pos:] diff --git a/src/postmaster/whatsapp_v990/groups.py b/src/postmaster/whatsapp_v990/groups.py new file mode 100644 index 00000000..a29398b0 --- /dev/null +++ b/src/postmaster/whatsapp_v990/groups.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +"""Clean-room WhatsApp group metadata/listing helpers. + +The wire query follows the current public Web companion protocol: an IQ get to @g.us in +namespace w:g2 with a participating request asking for participants and description. +This module is read-only: it does not create, mutate, join, leave or send to groups. +""" + +from typing import Any + +from .binary import BinaryNode +from .jid import parse_jid + + +class WhatsAppGroupError(ValueError): + pass + + +def build_participating_groups_query() -> BinaryNode: + return BinaryNode( + "iq", + {"to": "@g.us", "xmlns": "w:g2", "type": "get"}, + [ + BinaryNode( + "participating", + {}, + [ + BinaryNode("participants"), + BinaryNode("description"), + ], + ) + ], + ) + + +def build_group_metadata_query(jid: str) -> BinaryNode: + parsed = parse_jid(str(jid)) + if not parsed.is_group: + raise WhatsAppGroupError("WhatsApp group metadata query requires a g.us JID") + return BinaryNode( + "iq", + {"to": str(parsed), "xmlns": "w:g2", "type": "get"}, + [BinaryNode("query", {"request": "interactive"})], + ) + + +def _text(node: BinaryNode | None) -> str | None: + if node is None: + return None + value = node.content + if isinstance(value, bytes): + try: + return value.decode("utf-8") + except UnicodeDecodeError: + return value.decode("utf-8", errors="replace") + if isinstance(value, str): + return value + return None + + +def _int(value: Any) -> int | None: + if value in (None, ""): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _normalized_group_id(raw: str) -> str: + value = str(raw or "").strip() + if not value: + raise WhatsAppGroupError("WhatsApp group metadata is missing id") + return value if "@" in value else f"{value}@g.us" + + +def parse_group_metadata(group: BinaryNode) -> dict[str, Any]: + if group.tag != "group": + raise WhatsAppGroupError(f"Expected WhatsApp group node, got {group.tag!r}") + + description = group.child("description") + desc_body = description.child("body") if description is not None else None + ephemeral = group.child("ephemeral") + + participants: list[dict[str, Any]] = [] + for participant in group.children("participant"): + jid = str(participant.attrs.get("jid") or "").strip() + if not jid: + continue + row: dict[str, Any] = {"id": jid} + if participant.attrs.get("phone_number"): + row["phone_number"] = participant.attrs["phone_number"] + if participant.attrs.get("lid"): + row["lid"] = participant.attrs["lid"] + username = participant.attrs.get("participant_username") or participant.attrs.get("username") + if username: + row["username"] = username + if participant.attrs.get("type"): + row["admin"] = participant.attrs["type"] + participants.append(row) + + group_id = _normalized_group_id(str(group.attrs.get("id") or "")) + return { + "id": group_id, + "subject": group.attrs.get("subject"), + "notify": group.attrs.get("notify"), + "addressing_mode": "lid" if group.attrs.get("addressing_mode") == "lid" else "pn", + "subject_owner": group.attrs.get("s_o"), + "subject_owner_pn": group.attrs.get("s_o_pn"), + "subject_owner_username": group.attrs.get("s_o_username"), + "subject_time": _int(group.attrs.get("s_t")), + "size": _int(group.attrs.get("size")) if group.attrs.get("size") is not None else len(participants), + "creation": _int(group.attrs.get("creation")), + "owner": group.attrs.get("creator"), + "owner_pn": group.attrs.get("creator_pn"), + "owner_username": group.attrs.get("creator_username"), + "owner_country_code": group.attrs.get("creator_country_code"), + "description": _text(desc_body), + "description_id": description.attrs.get("id") if description else None, + "description_owner": description.attrs.get("participant") if description else None, + "description_owner_pn": description.attrs.get("participant_pn") if description else None, + "description_owner_username": description.attrs.get("participant_username") if description else None, + "description_time": _int(description.attrs.get("t")) if description else None, + "linked_parent": group.child("linked_parent").attrs.get("jid") if group.child("linked_parent") else None, + "restrict": group.child("locked") is not None, + "announce": group.child("announcement") is not None, + "is_community": group.child("parent") is not None, + "is_community_announce": group.child("default_sub_group") is not None, + "join_approval_mode": group.child("membership_approval_mode") is not None, + "member_add_mode": _text(group.child("member_add_mode")) == "all_member_add", + "ephemeral_duration": _int(ephemeral.attrs.get("expiration")) if ephemeral else None, + "participants": participants, + } + + +def parse_participating_groups(response: BinaryNode) -> list[dict[str, Any]]: + error = response.child("error") + if error is not None: + code = error.attrs.get("code") or "unknown" + message = error.attrs.get("text") or "group listing failed" + raise WhatsAppGroupError(f"WhatsApp group listing failed ({code}): {message}") + groups = response.child("groups") + if groups is None: + return [] + result: list[dict[str, Any]] = [] + for group in groups.children("group"): + result.append(parse_group_metadata(group)) + return result + + +def parse_group_metadata_response(response: BinaryNode) -> dict[str, Any]: + error = response.child("error") + if error is not None: + code = error.attrs.get("code") or "unknown" + message = error.attrs.get("text") or "group metadata query failed" + raise WhatsAppGroupError(f"WhatsApp group metadata failed ({code}): {message}") + group = response.child("group") + if group is None: + raise WhatsAppGroupError("WhatsApp group metadata response has no group node") + return parse_group_metadata(group) + + +__all__ = [ + "WhatsAppGroupError", + "build_participating_groups_query", + "build_group_metadata_query", + "parse_group_metadata", + "parse_participating_groups", + "parse_group_metadata_response", +] diff --git a/src/postmaster/whatsapp_v990/handshake.py b/src/postmaster/whatsapp_v990/handshake.py new file mode 100644 index 00000000..42929ced --- /dev/null +++ b/src/postmaster/whatsapp_v990/handshake.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from .proto import ProtoError, decode_fields, field_bytes, field_message + + +class HandshakeError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class ClientHello: + ephemeral: bytes + static: bytes = b"" + payload: bytes = b"" + + +@dataclass(frozen=True, slots=True) +class ServerHello: + ephemeral: bytes + static: bytes = b"" + payload: bytes = b"" + + +@dataclass(frozen=True, slots=True) +class ClientFinish: + static: bytes + payload: bytes + + +@dataclass(frozen=True, slots=True) +class HandshakeMessage: + client_hello: ClientHello | None = None + server_hello: ServerHello | None = None + client_finish: ClientFinish | None = None + + +def _bytes_fields(raw: bytes) -> dict[int, bytes]: + out: dict[int, bytes] = {} + try: + fields = decode_fields(raw) + except ProtoError as exc: + raise HandshakeError(str(exc)) from exc + for field in fields: + if not isinstance(field.value, bytes): + raise HandshakeError(f"Handshake field {field.number} has non-bytes wire type") + out[field.number] = field.value + return out + + +def _encode_hello(value: ClientHello | ServerHello) -> bytes: + fields = [field_bytes(1, value.ephemeral)] + if value.static: + fields.append(field_bytes(2, value.static)) + if value.payload: + fields.append(field_bytes(3, value.payload)) + return b"".join(fields) + + +def encode_handshake(message: HandshakeMessage) -> bytes: + variants = [message.client_hello is not None, message.server_hello is not None, message.client_finish is not None] + if sum(variants) != 1: + raise HandshakeError("HandshakeMessage must contain exactly one handshake phase") + if message.client_hello is not None: + return field_message(2, [_encode_hello(message.client_hello)]) + if message.server_hello is not None: + return field_message(3, [_encode_hello(message.server_hello)]) + finish = message.client_finish + assert finish is not None + return field_message(4, [field_bytes(1, finish.static), field_bytes(2, finish.payload)]) + + +def decode_handshake(raw: bytes) -> HandshakeMessage: + top = _bytes_fields(bytes(raw)) + present = [number for number in (2, 3, 4) if number in top] + if len(present) != 1: + raise HandshakeError("HandshakeMessage must contain exactly one recognized phase") + number = present[0] + nested = _bytes_fields(top[number]) + if number in (2, 3): + ephemeral = nested.get(1, b"") + if len(ephemeral) != 32: + raise HandshakeError("Handshake ephemeral public key must be 32 bytes") + cls = ClientHello if number == 2 else ServerHello + hello = cls(ephemeral=ephemeral, static=nested.get(2, b""), payload=nested.get(3, b"")) + return HandshakeMessage(client_hello=hello) if number == 2 else HandshakeMessage(server_hello=hello) + if 1 not in nested or 2 not in nested: + raise HandshakeError("ClientFinish requires static and payload") + return HandshakeMessage(client_finish=ClientFinish(static=nested[1], payload=nested[2])) + + +def encode_client_hello(ephemeral_public: bytes) -> bytes: + if len(ephemeral_public) != 32: + raise HandshakeError("ClientHello ephemeral public key must be 32 bytes") + return encode_handshake(HandshakeMessage(client_hello=ClientHello(ephemeral=bytes(ephemeral_public)))) + + +def encode_client_finish(encrypted_static: bytes, encrypted_payload: bytes) -> bytes: + return encode_handshake( + HandshakeMessage(client_finish=ClientFinish(static=bytes(encrypted_static), payload=bytes(encrypted_payload))) + ) + + +__all__ = [ + "HandshakeError", "ClientHello", "ServerHello", "ClientFinish", "HandshakeMessage", + "encode_handshake", "decode_handshake", "encode_client_hello", "encode_client_finish", +] diff --git a/src/postmaster/whatsapp_v990/jid.py b/src/postmaster/whatsapp_v990/jid.py new file mode 100644 index 00000000..a9c02309 --- /dev/null +++ b/src/postmaster/whatsapp_v990/jid.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +class JIDError(ValueError): + pass + + +_DOMAIN_TYPES = { + "s.whatsapp.net": 0, + "c.us": 0, + "lid": 1, + "hosted": 128, + "hosted.lid": 129, +} + + +@dataclass(frozen=True, slots=True) +class JID: + user: str + server: str + device: int | None = None + agent: int | None = None + domain_type: int = 0 + + def __post_init__(self) -> None: + if not self.server or "@" in self.server: + raise JIDError("JID server is required and must not contain @") + if self.device is not None and not 0 <= int(self.device) <= 255: + raise JIDError("JID device must fit one byte") + if self.agent is not None and int(self.agent) < 0: + raise JIDError("JID agent must be non-negative") + if not 0 <= int(self.domain_type) <= 255: + raise JIDError("JID domain_type must fit one byte") + + @property + def bare(self) -> str: + return f"{self.user}@{self.server}" + + @property + def is_group(self) -> bool: + return self.server == "g.us" + + @property + def is_broadcast(self) -> bool: + return self.server == "broadcast" + + @property + def is_newsletter(self) -> bool: + return self.server == "newsletter" + + @property + def is_lid(self) -> bool: + return self.server in {"lid", "hosted.lid"} + + def normalized_user(self) -> "JID": + server = "s.whatsapp.net" if self.server == "c.us" else self.server + return JID(self.user, server, domain_type=_DOMAIN_TYPES.get(server, self.domain_type)) + + def __str__(self) -> str: + left = self.user + if self.agent not in (None, 0): + left += f"_{int(self.agent)}" + if self.device not in (None, 0): + left += f":{int(self.device)}" + return f"{left}@{self.server}" + + +def parse_jid(value: str) -> JID: + raw = str(value or "").strip() + pos = raw.find("@") + if pos < 0: + raise JIDError("JID must contain @") + left, server = raw[:pos], raw[pos + 1 :] + if not server: + raise JIDError("JID server is empty") + + device: int | None = None + agent: int | None = None + if ":" in left: + left, device_raw = left.rsplit(":", 1) + if not device_raw.isdigit(): + raise JIDError("JID device must be numeric") + device = int(device_raw) + if "_" in left: + user, agent_raw = left.rsplit("_", 1) + if agent_raw.isdigit(): + left = user + agent = int(agent_raw) + + return JID( + user=left, + server=server, + device=device, + agent=agent, + domain_type=_DOMAIN_TYPES.get(server, agent or 0), + ) + + +def encode_jid(user: str | int | None, server: str, device: int | None = None, agent: int | None = None) -> str: + return str(JID("" if user is None else str(user), str(server), device=device, agent=agent, domain_type=_DOMAIN_TYPES.get(str(server), agent or 0))) + + +def same_user(a: str | JID, b: str | JID) -> bool: + aa = a if isinstance(a, JID) else parse_jid(a) + bb = b if isinstance(b, JID) else parse_jid(b) + return aa.user == bb.user + + +def transfer_device(source: str | JID, target: str | JID) -> JID: + src = source if isinstance(source, JID) else parse_jid(source) + dst = target if isinstance(target, JID) else parse_jid(target) + return JID(dst.user, dst.server, device=src.device, agent=dst.agent, domain_type=dst.domain_type) diff --git a/src/postmaster/whatsapp_v990/media.py b/src/postmaster/whatsapp_v990/media.py new file mode 100644 index 00000000..5e8424e9 --- /dev/null +++ b/src/postmaster/whatsapp_v990/media.py @@ -0,0 +1,392 @@ +from __future__ import annotations + +"""Clean-room WhatsApp media upload/download and minimal media protobuf helpers.""" + +import base64 +from dataclasses import dataclass +import json +import time +from typing import Any +from urllib.parse import quote + +import httpx + +from .binary import BinaryNode +from .crypto import decrypt_media, encrypt_media +from .proto import ProtoError, decode_fields, field_bytes, field_message, field_varint + + +class WhatsAppMediaError(RuntimeError): + pass + + +MEDIA_PATH_MAP = { + "image": "/mms/image", + "video": "/mms/video", + "audio": "/mms/audio", + "document": "/mms/document", + "sticker": "/mms/image", +} +OUTER_MESSAGE_FIELD = {"image": 3, "document": 7, "audio": 8, "video": 9, "sticker": 26} + + +@dataclass(frozen=True, slots=True) +class MediaHost: + hostname: str + max_content_length_bytes: int | None = None + + +@dataclass(frozen=True, slots=True) +class MediaConn: + auth: str + ttl: int + hosts: tuple[MediaHost, ...] + + +@dataclass(frozen=True, slots=True) +class MediaUpload: + media_type: str + url: str | None + direct_path: str | None + media_key: bytes + file_sha256: bytes + file_enc_sha256: bytes + file_length: int + mimetype: str + filename: str | None = None + caption: str = "" + media_key_timestamp: int = 0 + + +@dataclass(frozen=True, slots=True) +class MediaDescriptor: + media_type: str + url: str | None + direct_path: str | None + media_key: bytes + file_sha256: bytes + file_enc_sha256: bytes + file_length: int + mimetype: str + filename: str | None = None + caption: str = "" + + +def infer_media_type(*, mimetype: str, filename: str = "") -> str: + mime = str(mimetype or "").split(";", 1)[0].strip().lower() + name = str(filename or "").lower() + if mime == "image/webp" or name.endswith(".webp"): + return "sticker" + if mime.startswith("image/"): + return "image" + if mime.startswith("video/"): + return "video" + if mime.startswith("audio/"): + return "audio" + return "document" + + +def build_media_conn_query() -> BinaryNode: + return BinaryNode( + "iq", + {"type": "set", "xmlns": "w:m", "to": "s.whatsapp.net"}, + [BinaryNode("media_conn")], + ) + + +def parse_media_conn(node: BinaryNode) -> MediaConn: + media = node.child("media_conn") if node.tag == "iq" else None + if media is None: + raise WhatsAppMediaError("WhatsApp media_conn response is missing media_conn") + auth = str(media.attrs.get("auth") or "").strip() + if not auth: + raise WhatsAppMediaError("WhatsApp media_conn response has no auth token") + try: + ttl = int(media.attrs.get("ttl") or 0) + except ValueError as exc: + raise WhatsAppMediaError("WhatsApp media_conn ttl is invalid") from exc + hosts: list[MediaHost] = [] + for child in media.children("host"): + hostname = str(child.attrs.get("hostname") or "").strip() + if not hostname or "/" in hostname or ":" in hostname: + continue + raw_limit = child.attrs.get("maxContentLengthBytes") + try: + limit = int(raw_limit) if raw_limit not in (None, "") else None + except (TypeError, ValueError): + limit = None + hosts.append(MediaHost(hostname, limit)) + if not hosts: + raise WhatsAppMediaError("WhatsApp media_conn response has no usable upload hosts") + return MediaConn(auth=auth, ttl=max(0, ttl), hosts=tuple(hosts)) + + +def upload_token(file_enc_sha256: bytes) -> str: + # URL-safe unpadded Base64, matching the current WhatsApp Web upload token. + value = base64.b64encode(bytes(file_enc_sha256)).decode("ascii") + value = value.replace("+", "-").replace("/", "_").rstrip("=") + return quote(value, safe="-_") + + +async def upload_media_bytes( + plaintext: bytes, + *, + media_type: str, + mimetype: str, + filename: str | None, + caption: str = "", + media_conn: MediaConn, + client: httpx.AsyncClient | None = None, + timeout_seconds: float = 30.0, +) -> tuple[MediaUpload, bytes]: + kind = str(media_type).lower() + if kind not in MEDIA_PATH_MAP: + raise WhatsAppMediaError(f"Unsupported WhatsApp media type: {media_type}") + data = bytes(plaintext) + encrypted = encrypt_media(data, kind) + token = upload_token(encrypted["file_enc_sha256"]) + own_client = client is None + http = client or httpx.AsyncClient( + timeout=httpx.Timeout(timeout_seconds), + follow_redirects=True, + headers={"Origin": "https://web.whatsapp.com"}, + ) + last_error: Exception | None = None + try: + for host in media_conn.hosts: + if host.max_content_length_bytes is not None and len(data) > host.max_content_length_bytes: + last_error = WhatsAppMediaError( + f"Stored File exceeds WhatsApp upload host limit ({host.max_content_length_bytes} bytes)" + ) + continue + url = ( + f"https://{host.hostname}{MEDIA_PATH_MAP[kind]}/{token}" + f"?auth={quote(media_conn.auth, safe='')}&token={token}" + ) + try: + response = await http.post( + url, + content=encrypted["encrypted"], + headers={ + "Content-Type": "application/octet-stream", + "Origin": "https://web.whatsapp.com", + }, + ) + response.raise_for_status() + value = response.json() + if not isinstance(value, dict): + raise WhatsAppMediaError("WhatsApp media upload returned non-object JSON") + media_url = str(value.get("url") or "").strip() or None + direct_path = str(value.get("direct_path") or "").strip() or None + if not media_url and not direct_path: + raise WhatsAppMediaError("WhatsApp media upload returned no URL/direct path") + return ( + MediaUpload( + media_type=kind, + url=media_url, + direct_path=direct_path, + media_key=encrypted["media_key"], + file_sha256=encrypted["file_sha256"], + file_enc_sha256=encrypted["file_enc_sha256"], + file_length=len(data), + mimetype=str(mimetype or "application/octet-stream"), + filename=str(filename) if filename else None, + caption=str(caption or ""), + media_key_timestamp=int(time.time()), + ), + encrypted["encrypted"], + ) + except Exception as exc: + last_error = exc + raise WhatsAppMediaError(f"WhatsApp media upload failed on all hosts: {last_error}") + finally: + if own_client: + await http.aclose() + + +def encode_media_message(upload: MediaUpload) -> bytes: + kind = upload.media_type + if kind not in OUTER_MESSAGE_FIELD: + raise WhatsAppMediaError(f"Unsupported media protobuf type: {kind}") + common_url = [field_bytes(1, upload.url)] if upload.url else [] + + if kind == "image": + inner = common_url + [ + field_bytes(2, upload.mimetype), + *([field_bytes(3, upload.caption)] if upload.caption else []), + field_bytes(4, upload.file_sha256), + field_varint(5, upload.file_length), + field_bytes(8, upload.media_key), + field_bytes(9, upload.file_enc_sha256), + *([field_bytes(11, upload.direct_path)] if upload.direct_path else []), + field_varint(12, upload.media_key_timestamp), + ] + elif kind == "document": + inner = common_url + [ + field_bytes(2, upload.mimetype), + field_bytes(4, upload.file_sha256), + field_varint(5, upload.file_length), + field_bytes(7, upload.media_key), + *([field_bytes(8, upload.filename)] if upload.filename else []), + field_bytes(9, upload.file_enc_sha256), + *([field_bytes(10, upload.direct_path)] if upload.direct_path else []), + field_varint(11, upload.media_key_timestamp), + *([field_bytes(20, upload.caption)] if upload.caption else []), + ] + elif kind == "audio": + inner = common_url + [ + field_bytes(2, upload.mimetype), + field_bytes(3, upload.file_sha256), + field_varint(4, upload.file_length), + field_bytes(7, upload.media_key), + field_bytes(8, upload.file_enc_sha256), + *([field_bytes(9, upload.direct_path)] if upload.direct_path else []), + field_varint(10, upload.media_key_timestamp), + ] + elif kind == "video": + inner = common_url + [ + field_bytes(2, upload.mimetype), + field_bytes(3, upload.file_sha256), + field_varint(4, upload.file_length), + field_bytes(6, upload.media_key), + *([field_bytes(7, upload.caption)] if upload.caption else []), + field_bytes(11, upload.file_enc_sha256), + *([field_bytes(13, upload.direct_path)] if upload.direct_path else []), + field_varint(14, upload.media_key_timestamp), + ] + else: # sticker + inner = common_url + [ + field_bytes(2, upload.file_sha256), + field_bytes(3, upload.file_enc_sha256), + field_bytes(4, upload.media_key), + field_bytes(5, upload.mimetype), + *([field_bytes(8, upload.direct_path)] if upload.direct_path else []), + field_varint(9, upload.file_length), + field_varint(10, upload.media_key_timestamp), + ] + return field_message(OUTER_MESSAGE_FIELD[kind], inner) + + +def _field_map(raw: bytes) -> dict[int, list[int | bytes]]: + try: + fields = decode_fields(raw) + except ProtoError as exc: + raise WhatsAppMediaError("Invalid WhatsApp media protobuf") from exc + out: dict[int, list[int | bytes]] = {} + for field in fields: + out.setdefault(field.number, []).append(field.value) + return out + + +def _bytes(values: dict[int, list[int | bytes]], number: int) -> bytes | None: + found = [value for value in values.get(number, []) if isinstance(value, bytes)] + return bytes(found[-1]) if found else None + + +def _integer(values: dict[int, list[int | bytes]], number: int) -> int | None: + found = [value for value in values.get(number, []) if isinstance(value, int)] + return int(found[-1]) if found else None + + +def _text(values: dict[int, list[int | bytes]], number: int) -> str | None: + raw = _bytes(values, number) + if raw is None: + return None + try: + return raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise WhatsAppMediaError("WhatsApp media protobuf string is not UTF-8") from exc + + +def decode_media_descriptor(message: bytes) -> MediaDescriptor | None: + outer = _field_map(bytes(message)) + reverse = {value: key for key, value in OUTER_MESSAGE_FIELD.items()} + selected: tuple[str, bytes] | None = None + for field_number, kind in reverse.items(): + nested = _bytes(outer, field_number) + if nested is not None: + selected = (kind, nested) + break + if selected is None: + # WAProto.Message.deviceSentMessage = field 31; DeviceSentMessage.message = field 2. + wrapper = _bytes(outer, 31) + if wrapper is not None: + wrapped = _field_map(wrapper) + nested_message = _bytes(wrapped, 2) + if nested_message is not None: + return decode_media_descriptor(nested_message) + return None + kind, nested = selected + values = _field_map(nested) + if kind == "image": + sha, length, key, enc, direct, caption, filename = 4, 5, 8, 9, 11, 3, None + elif kind == "document": + sha, length, key, enc, direct, caption, filename = 4, 5, 7, 9, 10, 20, 8 + elif kind == "audio": + sha, length, key, enc, direct, caption, filename = 3, 4, 7, 8, 9, None, None + elif kind == "video": + sha, length, key, enc, direct, caption, filename = 3, 4, 6, 11, 13, 7, None + else: + sha, length, key, enc, direct, caption, filename = 2, 9, 4, 3, 8, None, None + media_key = _bytes(values, key) + file_sha = _bytes(values, sha) + file_enc_sha = _bytes(values, enc) + file_length = _integer(values, length) + if media_key is None or file_sha is None or file_enc_sha is None or file_length is None: + raise WhatsAppMediaError("WhatsApp media message is missing cryptographic metadata") + return MediaDescriptor( + media_type=kind, + url=_text(values, 1), + direct_path=_text(values, direct), + media_key=media_key, + file_sha256=file_sha, + file_enc_sha256=file_enc_sha, + file_length=file_length, + mimetype=_text(values, 2 if kind != "sticker" else 5) or "application/octet-stream", + filename=_text(values, filename) if filename else None, + caption=_text(values, caption) if caption else "", + ) + + +async def download_media( + descriptor: MediaDescriptor, + *, + client: httpx.AsyncClient | None = None, + default_host: str = "mmg.whatsapp.net", + timeout_seconds: float = 30.0, +) -> bytes: + if descriptor.direct_path: + url = f"https://{default_host}{descriptor.direct_path}" + elif descriptor.url: + url = descriptor.url + else: + raise WhatsAppMediaError("WhatsApp media message has no URL/direct path") + if not url.startswith("https://"): + raise WhatsAppMediaError("WhatsApp media download URL must use HTTPS") + own_client = client is None + http = client or httpx.AsyncClient(timeout=httpx.Timeout(timeout_seconds), follow_redirects=True) + try: + response = await http.get(url, headers={"Origin": "https://web.whatsapp.com"}) + response.raise_for_status() + encrypted = bytes(response.content) + if __import__("hashlib").sha256(encrypted).digest() != descriptor.file_enc_sha256: + raise WhatsAppMediaError("Downloaded WhatsApp media encrypted SHA-256 mismatch") + plaintext = decrypt_media( + encrypted, + descriptor.media_type, + descriptor.media_key, + expected_file_sha256=descriptor.file_sha256, + ) + if len(plaintext) != descriptor.file_length: + raise WhatsAppMediaError("Downloaded WhatsApp media length mismatch") + return plaintext + finally: + if own_client: + await http.aclose() + + +__all__ = [ + "WhatsAppMediaError", "MediaHost", "MediaConn", "MediaUpload", "MediaDescriptor", + "infer_media_type", "build_media_conn_query", "parse_media_conn", "upload_token", + "upload_media_bytes", "encode_media_message", "decode_media_descriptor", "download_media", +] diff --git a/src/postmaster/whatsapp_v990/messages.py b/src/postmaster/whatsapp_v990/messages.py new file mode 100644 index 00000000..3af90a84 --- /dev/null +++ b/src/postmaster/whatsapp_v990/messages.py @@ -0,0 +1,265 @@ +from __future__ import annotations + +"""Minimal WhatsApp 1:1 message protobuf and WABinary stanza builders.""" + +import base64 +import hashlib +import os +import time +from typing import Iterable + +from .binary import BinaryNode +from .jid import parse_jid +from .proto import ProtoError, decode_fields, field_bytes, field_message + + +class WhatsAppMessageError(ValueError): + pass + + +def pad_random_max16(message: bytes, *, random1: bytes | None = None) -> bytes: + """Apply WhatsApp's 1..16 byte message padding before Signal encryption.""" + entropy = bytes(random1 if random1 is not None else os.urandom(1)) + if len(entropy) != 1: + raise WhatsAppMessageError("WhatsApp message padding entropy must be exactly one byte") + pad_length = (entropy[0] & 0x0F) + 1 + return bytes(message) + bytes((pad_length,)) * pad_length + + +def unpad_random_max16(message: bytes) -> bytes: + """Remove WhatsApp message padding after Signal decryption.""" + raw = bytes(message) + if not raw: + raise WhatsAppMessageError("Cannot unpad an empty WhatsApp message") + pad_length = raw[-1] + if pad_length < 1 or pad_length > 16 or pad_length > len(raw): + raise WhatsAppMessageError("Invalid WhatsApp message padding") + return raw[:-pad_length] + + +def generate_message_id_v2( + user_jid: str | None = None, + *, + now_seconds: int | None = None, + random16: bytes | None = None, +) -> str: + data = bytearray(44) + timestamp = int(time.time() if now_seconds is None else now_seconds) + if timestamp < 0: + raise WhatsAppMessageError("Message timestamp must be non-negative") + data[:8] = timestamp.to_bytes(8, "big", signed=False) + if user_jid: + user = parse_jid(user_jid).user.encode("utf-8") + identity = user + b"@c.us" + data[8:8 + min(20, len(identity))] = identity[:20] + rnd = bytes(random16 if random16 is not None else os.urandom(16)) + if len(rnd) != 16: + raise WhatsAppMessageError("Message ID entropy must be 16 bytes") + data[28:44] = rnd + digest = hashlib.sha256(data).hexdigest().upper() + return "3EB0" + digest[:18] + + +def participant_hash_v2(participants: Iterable[str]) -> str: + normalized = sorted(str(x) for x in participants if str(x)) + digest = hashlib.sha256("".join(normalized).encode("utf-8")).digest() + return "2:" + base64.b64encode(digest).decode("ascii")[:6] + + +def encode_text_message(text: str) -> bytes: + value = str(text) + if not value: + raise WhatsAppMessageError("WhatsApp text message cannot be empty") + # WAProto.Message.conversation = field 1. + return field_bytes(1, value) + + +def encode_reply_text_message( + text: str, + *, + stanza_id: str, + participant: str, + remote_jid: str | None = None, + quoted_message: bytes | None = None, +) -> bytes: + value = str(text) + reply_id = str(stanza_id or "").strip() + sender = str(parse_jid(participant)) + if not value: + raise WhatsAppMessageError("WhatsApp reply text cannot be empty") + if not reply_id: + raise WhatsAppMessageError("WhatsApp reply requires quoted stanza id") + context = [field_bytes(1, reply_id), field_bytes(2, sender)] + if quoted_message is not None: + context.append(field_bytes(3, bytes(quoted_message))) + if remote_jid: + context.append(field_bytes(4, str(parse_jid(remote_jid).normalized_user()))) + extended = [field_bytes(1, value), field_message(17, context)] + # WAProto.Message.extendedTextMessage = field 6. + return field_message(6, extended) + + +def decode_text_message(message: bytes) -> str | None: + """Extract the text subset supported by the v9.9 clean-room runtime. + + Supports conversation, extendedTextMessage and DeviceSentMessage wrappers. Unknown + message kinds return None rather than being mis-decoded as text. + """ + try: + fields = decode_fields(bytes(message)) + except ProtoError as exc: + raise WhatsAppMessageError("Invalid WhatsApp Message protobuf") from exc + for field in fields: + if field.number == 1 and isinstance(field.value, bytes): + try: + return field.value.decode("utf-8") + except UnicodeDecodeError as exc: + raise WhatsAppMessageError("WhatsApp conversation text is not UTF-8") from exc + if field.number == 6 and isinstance(field.value, bytes): + try: + extended = decode_fields(field.value) + except ProtoError as exc: + raise WhatsAppMessageError("Invalid WhatsApp extendedTextMessage protobuf") from exc + text_field = next((item for item in extended if item.number == 1 and isinstance(item.value, bytes)), None) + if text_field is not None: + try: + return bytes(text_field.value).decode("utf-8") + except UnicodeDecodeError as exc: + raise WhatsAppMessageError("WhatsApp extended text is not UTF-8") from exc + if field.number == 31 and isinstance(field.value, bytes): + try: + device_sent = decode_fields(field.value) + except ProtoError as exc: + raise WhatsAppMessageError("Invalid WhatsApp DeviceSentMessage protobuf") from exc + nested = next((item for item in device_sent if item.number == 2 and isinstance(item.value, bytes)), None) + if nested is not None: + return decode_text_message(bytes(nested.value)) + return None + + +def build_ack_stanza(node: BinaryNode, *, me_id: str | None = None, error_code: int | None = None) -> BinaryNode: + """Build the protocol transport ACK for a received stanza. + + This is never a WhatsApp read receipt. In particular, message ACKs acknowledge delivery + to this companion transport without marking the conversation as read. + """ + stanza_id = str(node.attrs.get("id") or "").strip() + sender = str(node.attrs.get("from") or "").strip() + if not stanza_id or not sender: + raise WhatsAppMessageError("WhatsApp ACK requires received stanza id and from") + attrs = {"id": stanza_id, "to": sender, "class": node.tag} + if error_code: + attrs["error"] = str(int(error_code)) + for name in ("participant", "recipient", "type"): + value = node.attrs.get(name) + if value: + attrs[name] = str(value) + if node.tag == "message" and me_id: + attrs["from"] = str(me_id) + return BinaryNode("ack", attrs) + + +def build_read_receipt(*, destination_jid: str, message_id: str) -> BinaryNode: + destination = str(parse_jid(destination_jid).normalized_user()) + mid = str(message_id or "").strip() + if not mid: + raise WhatsAppMessageError("WhatsApp read receipt requires message id") + return BinaryNode("receipt", {"id": mid, "to": destination, "type": "read"}) + + +def encode_device_sent_message(destination_jid: str, message: bytes, *, phash: str | None = None) -> bytes: + destination = str(parse_jid(destination_jid)) + fields = [field_bytes(1, destination), field_bytes(2, bytes(message))] + if phash: + fields.append(field_bytes(3, str(phash))) + # WAProto.Message.deviceSentMessage = field 31. + return field_message(31, fields) + + +def encrypted_participant_node(jid: str, *, ciphertext_type: str, ciphertext: bytes, extra_attrs: dict[str, str] | None = None) -> BinaryNode: + target = str(parse_jid(jid)) + kind = str(ciphertext_type) + if kind not in {"msg", "pkmsg"}: + raise WhatsAppMessageError("Direct Signal ciphertext type must be msg or pkmsg") + attrs = {"v": "2", "type": kind} + attrs.update({str(k): str(v) for k, v in (extra_attrs or {}).items()}) + return BinaryNode("to", {"jid": target}, [BinaryNode("enc", attrs, bytes(ciphertext))]) + + +def build_group_message_stanza( + *, + destination_jid: str, + message_id: str, + sender_key_ciphertext: bytes, + sender_key_recipients: Iterable[BinaryNode] = (), + device_identity: bytes | None = None, + message_type: str = "text", + addressing_mode: str = "lid", + additional_attrs: dict[str, str] | None = None, + enc_attrs: dict[str, str] | None = None, +) -> BinaryNode: + destination = parse_jid(destination_jid) + if not destination.is_group: + raise WhatsAppMessageError("Group WhatsApp stanza requires a g.us destination") + if addressing_mode not in {"lid", "pn"}: + raise WhatsAppMessageError("WhatsApp group addressing_mode must be lid or pn") + ciphertext = bytes(sender_key_ciphertext) + if not ciphertext: + raise WhatsAppMessageError("Group WhatsApp stanza requires sender-key ciphertext") + + attrs = { + "id": str(message_id), + "to": str(destination), + "type": str(message_type), + "addressing_mode": addressing_mode, + } + attrs.update({str(k): str(v) for k, v in (additional_attrs or {}).items()}) + + content: list[BinaryNode] = [] + recipients = list(sender_key_recipients) + if recipients: + if any(node.tag != "to" or not node.attrs.get("jid") for node in recipients): + raise WhatsAppMessageError("Group sender-key recipient nodes must be ") + content.append(BinaryNode("participants", {}, recipients)) + if device_identity is not None: + content.append(BinaryNode("device-identity", {}, bytes(device_identity))) + encryption_attrs = {"v": "2", "type": "skmsg"} + encryption_attrs.update({str(k): str(v) for k, v in (enc_attrs or {}).items()}) + content.append(BinaryNode("enc", encryption_attrs, ciphertext)) + return BinaryNode("message", attrs, content) + + +def build_direct_message_stanza( + *, + destination_jid: str, + message_id: str, + participants: Iterable[BinaryNode], + device_identity: bytes | None = None, + message_type: str = "text", + additional_attrs: dict[str, str] | None = None, +) -> BinaryNode: + destination = str(parse_jid(destination_jid).normalized_user()) + nodes = list(participants) + if not nodes: + raise WhatsAppMessageError("Direct WhatsApp message requires at least one encrypted participant") + participant_jids = [str(node.attrs.get("jid") or "") for node in nodes] + if any(not jid for jid in participant_jids): + raise WhatsAppMessageError("Encrypted participant is missing jid") + attrs = { + "id": str(message_id), + "to": destination, + "type": str(message_type), + "phash": participant_hash_v2(participant_jids), + } + attrs.update({str(k): str(v) for k, v in (additional_attrs or {}).items()}) + content: list[BinaryNode] = [BinaryNode("participants", {}, nodes)] + if device_identity is not None: + content.append(BinaryNode("device-identity", {}, bytes(device_identity))) + return BinaryNode("message", attrs, content) + + +__all__ = [ + "WhatsAppMessageError", "pad_random_max16", "unpad_random_max16", "generate_message_id_v2", "participant_hash_v2", + "encode_text_message", "encode_reply_text_message", "decode_text_message", "build_ack_stanza", "build_read_receipt", + "encode_device_sent_message", "encrypted_participant_node", "build_group_message_stanza", "build_direct_message_stanza", +] diff --git a/src/postmaster/whatsapp_v990/pairing.py b/src/postmaster/whatsapp_v990/pairing.py new file mode 100644 index 00000000..ad1ab2e0 --- /dev/null +++ b/src/postmaster/whatsapp_v990/pairing.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import base64 +from dataclasses import dataclass + + +PAIRING_QR_PREFIX = "https://wa.me/settings/linked_devices#" + + +class PairingError(ValueError): + pass + + +_BROWSER_TYPES = { + "chrome": 1, + "edge": 2, + "firefox": 3, + "ie": 4, + "opera": 5, + "safari": 6, +} + + +def companion_web_client_type(os_name: str, browser_name: str) -> int: + os_name = str(os_name or "").strip().lower() + browser_name = str(browser_name or "").strip() + if browser_name.lower() == "desktop": + return 8 if os_name == "windows" else 7 + return _BROWSER_TYPES.get(browser_name.lower(), 9) + + +def _b64(raw: bytes) -> str: + return base64.b64encode(bytes(raw)).decode("ascii") + + +def build_pairing_qr_data( + ref: str, + noise_public_key: bytes, + identity_public_key: bytes, + adv_secret_key: bytes | str, + *, + os_name: str = "Linux", + browser_name: str = "Chrome", +) -> str: + ref = str(ref or "").strip() + if not ref: + raise PairingError("Pairing reference is required") + if len(noise_public_key) != 32 or len(identity_public_key) != 32: + raise PairingError("Pairing public keys must be 32 bytes") + adv = adv_secret_key if isinstance(adv_secret_key, str) else _b64(adv_secret_key) + if not str(adv).strip(): + raise PairingError("ADV secret is required for companion QR") + fields = [ + ref, + _b64(noise_public_key), + _b64(identity_public_key), + str(adv), + str(companion_web_client_type(os_name, browser_name)), + ] + return PAIRING_QR_PREFIX + ",".join(fields) + + +@dataclass(frozen=True, slots=True) +class PairingQR: + payload: str + expires_at: str | None = None + + def public_status(self) -> dict[str, object]: + # Payload is intentionally omitted from generic status; it belongs only to the explicit + # pairing response/WebGUI view and is persisted encrypted when cached. + return {"available": bool(self.payload), "expires_at": self.expires_at, "payload_exposed_in_status": False} + + +__all__ = [ + "PAIRING_QR_PREFIX", "PairingError", "PairingQR", "companion_web_client_type", "build_pairing_qr_data", +] diff --git a/src/postmaster/whatsapp_v990/proto.py b/src/postmaster/whatsapp_v990/proto.py new file mode 100644 index 00000000..506d7b31 --- /dev/null +++ b/src/postmaster/whatsapp_v990/proto.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Iterable + + +class ProtoError(ValueError): + pass + + +WIRE_VARINT = 0 +WIRE_64BIT = 1 +WIRE_LENGTH = 2 +WIRE_32BIT = 5 + + +def encode_varint(value: int) -> bytes: + value = int(value) + if value < 0: + value &= (1 << 64) - 1 + out = bytearray() + while value > 0x7F: + out.append((value & 0x7F) | 0x80); value >>= 7 + out.append(value) + return bytes(out) + + +def decode_varint(data: bytes, offset: int = 0) -> tuple[int, int]: + result = 0; shift = 0; pos = int(offset) + while True: + if pos >= len(data): raise ProtoError("Truncated protobuf varint") + byte = data[pos]; pos += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): return result, pos + shift += 7 + if shift >= 70: raise ProtoError("Oversized protobuf varint") + + +@dataclass(frozen=True, slots=True) +class ProtoField: + number: int + wire_type: int + value: int | bytes + + +def field_varint(number: int, value: int) -> bytes: + return encode_varint((int(number) << 3) | WIRE_VARINT) + encode_varint(value) + + +def field_bytes(number: int, value: bytes | bytearray | memoryview | str) -> bytes: + raw = value.encode("utf-8") if isinstance(value, str) else bytes(value) + return encode_varint((int(number) << 3) | WIRE_LENGTH) + encode_varint(len(raw)) + raw + + +def field_message(number: int, fields: Iterable[bytes]) -> bytes: + return field_bytes(number, b"".join(fields)) + + +def decode_fields(data: bytes) -> list[ProtoField]: + raw = bytes(data); pos = 0; fields: list[ProtoField] = [] + while pos < len(raw): + key, pos = decode_varint(raw, pos) + number, wire = key >> 3, key & 7 + if number <= 0: raise ProtoError("Invalid protobuf field number") + if wire == WIRE_VARINT: + value, pos = decode_varint(raw, pos) + elif wire == WIRE_LENGTH: + length, pos = decode_varint(raw, pos) + if pos + length > len(raw): raise ProtoError("Truncated protobuf length-delimited field") + value = raw[pos:pos+length]; pos += length + elif wire == WIRE_64BIT: + if pos + 8 > len(raw): raise ProtoError("Truncated protobuf fixed64 field") + value = raw[pos:pos+8]; pos += 8 + elif wire == WIRE_32BIT: + if pos + 4 > len(raw): raise ProtoError("Truncated protobuf fixed32 field") + value = raw[pos:pos+4]; pos += 4 + else: + raise ProtoError(f"Unsupported protobuf wire type {wire}") + fields.append(ProtoField(number, wire, value)) + return fields diff --git a/src/postmaster/whatsapp_v990/receipts.py b/src/postmaster/whatsapp_v990/receipts.py new file mode 100644 index 00000000..4bde397c --- /dev/null +++ b/src/postmaster/whatsapp_v990/receipts.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +class ReceiptPolicyError(PermissionError): + pass + + +@dataclass(frozen=True, slots=True) +class ReceiptDecision: + emit: bool + receipt_type: str | None + reason: str + + +def receipt_for_event(*, event: str, outbound_action: bool = False, reply_to_message: bool = False) -> ReceiptDecision: + """Enforce Postmaster's asymmetric WhatsApp receipt contract. + + Merely displaying/reading/syncing a message never emits a read receipt. The only permitted + read receipt path is coupled to an explicit outbound send/reply flow. + """ + normalized = str(event or "").strip().lower() + if normalized in {"display", "read", "sync", "history", "inspect"}: + return ReceiptDecision(False, None, "local_read_is_private") + if normalized in {"send", "reply"}: + if outbound_action and (normalized == "reply" or reply_to_message): + return ReceiptDecision(True, "read", "coupled_to_explicit_outbound_reply") + return ReceiptDecision(False, None, "outbound_send_without_reply_does_not_imply_read") + return ReceiptDecision(False, None, "no_receipt_for_event") + + +def assert_can_emit_read_receipt(*, outbound_action: bool, reply_to_message: bool) -> None: + decision = receipt_for_event(event="reply", outbound_action=outbound_action, reply_to_message=reply_to_message) + if not decision.emit: + raise ReceiptPolicyError("WhatsApp read receipt is allowed only during an explicit outbound reply flow") diff --git a/src/postmaster/whatsapp_v990/sender_key.py b/src/postmaster/whatsapp_v990/sender_key.py new file mode 100644 index 00000000..d8344a5a --- /dev/null +++ b/src/postmaster/whatsapp_v990/sender_key.py @@ -0,0 +1,424 @@ +from __future__ import annotations + +"""Clean-room Signal SenderKey v3 primitives for WhatsApp group messaging. + +Wire format and derivation follow the current public Signal/WhatsApp sender-key protocol: +- version byte 0x33, +- HMAC-SHA256 chain/message seeds, +- HKDF-SHA256 info "WhisperGroup", +- AES-CBC with PKCS#7, +- XEdDSA signatures, +- bounded skipped-message-key cache. + +State is persisted through Postmaster's encrypted auth store and never logged. +""" + +from dataclasses import dataclass, field +import base64 +import hashlib +import hmac +import os +import secrets +from typing import Any + +from .crypto import ( + CurveKeyPair, + aes_cbc_decrypt, + aes_cbc_encrypt, + generate_curve_keypair, + hkdf_sha256, + signal_public_key, + xeddsa_sign, + xeddsa_verify, +) +from .proto import ProtoError, decode_fields, field_bytes, field_message, field_varint +from .store import EncryptedAuthStore + +SENDER_KEY_VERSION = 3 +SENDER_KEY_VERSION_BYTE = 0x33 +MAX_SENDER_MESSAGE_KEYS = 2000 +MAX_SENDER_KEY_STATES = 5 + + +class SenderKeyError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class SenderMessageKey: + iteration: int + seed: bytes + + @property + def material(self) -> bytes: + if len(self.seed) != 32: + raise SenderKeyError("Sender message-key seed must be 32 bytes") + return hkdf_sha256(self.seed, 64, salt=bytes(32), info=b"WhisperGroup") + + @property + def iv(self) -> bytes: + return self.material[:16] + + @property + def cipher_key(self) -> bytes: + return self.material[16:48] + + +@dataclass(slots=True) +class SenderKeyState: + key_id: int + iteration: int + chain_key: bytes + signing_public: bytes + signing_private: bytes | None = None + skipped: dict[int, bytes] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not 0 <= int(self.key_id) <= 0x7FFFFFFF: + raise SenderKeyError("Sender-key id must fit signed 31 bits") + if int(self.iteration) < 0: + raise SenderKeyError("Sender-key iteration must be non-negative") + if len(self.chain_key) != 32: + raise SenderKeyError("Sender chain key must be 32 bytes") + public = bytes(self.signing_public) + if len(public) == 33 and public[0] == 5: + public = public[1:] + if len(public) != 32: + raise SenderKeyError("Sender signing public key must be 32 bytes or 0x05-prefixed 33 bytes") + self.signing_public = public + if self.signing_private is not None and len(self.signing_private) != 32: + raise SenderKeyError("Sender signing private key must be 32 bytes") + + +@dataclass(frozen=True, slots=True) +class SenderKeyDistribution: + key_id: int + iteration: int + chain_key: bytes + signing_public: bytes + + def serialize(self) -> bytes: + public = signal_public_key(self.signing_public) + body = b"".join( + ( + field_varint(1, self.key_id), + field_varint(2, self.iteration), + field_bytes(3, self.chain_key), + field_bytes(4, public), + ) + ) + return bytes((SENDER_KEY_VERSION_BYTE,)) + body + + @classmethod + def parse(cls, raw: bytes) -> "SenderKeyDistribution": + data = bytes(raw) + if len(data) < 2 or data[0] != SENDER_KEY_VERSION_BYTE: + raise SenderKeyError("Unsupported sender-key distribution version") + fields = _field_map(data[1:]) + key_id = _required_int(fields, 1, "id") + iteration = _required_int(fields, 2, "iteration") + chain = _required_bytes(fields, 3, "chainKey") + public = _required_bytes(fields, 4, "signingKey") + if len(chain) != 32: + raise SenderKeyError("Sender-key distribution chain key must be 32 bytes") + if len(public) == 33 and public[0] == 5: + public = public[1:] + if len(public) != 32: + raise SenderKeyError("Sender-key distribution signing key has invalid length") + return cls(key_id, iteration, chain, public) + + +@dataclass(frozen=True, slots=True) +class SenderKeyMessage: + key_id: int + iteration: int + ciphertext: bytes + signature: bytes + + @property + def unsigned(self) -> bytes: + body = b"".join( + ( + field_varint(1, self.key_id), + field_varint(2, self.iteration), + field_bytes(3, self.ciphertext), + ) + ) + return bytes((SENDER_KEY_VERSION_BYTE,)) + body + + def serialize(self) -> bytes: + if len(self.signature) != 64: + raise SenderKeyError("Sender-key message signature must be 64 bytes") + return self.unsigned + self.signature + + @classmethod + def create(cls, *, key_id: int, iteration: int, ciphertext: bytes, signing_private: bytes) -> "SenderKeyMessage": + placeholder = cls(key_id, iteration, bytes(ciphertext), b"") + signature = xeddsa_sign(signing_private, placeholder.unsigned) + return cls(key_id, iteration, bytes(ciphertext), signature) + + @classmethod + def parse(cls, raw: bytes) -> "SenderKeyMessage": + data = bytes(raw) + if len(data) < 1 + 64 or data[0] != SENDER_KEY_VERSION_BYTE: + raise SenderKeyError("Unsupported or truncated sender-key message") + body, signature = data[1:-64], data[-64:] + fields = _field_map(body) + return cls( + _required_int(fields, 1, "id"), + _required_int(fields, 2, "iteration"), + _required_bytes(fields, 3, "ciphertext"), + signature, + ) + + def verify(self, signing_public: bytes) -> bool: + return xeddsa_verify(signing_public, self.unsigned, self.signature) + + +def _field_map(raw: bytes) -> dict[int, list[int | bytes]]: + try: + fields = decode_fields(bytes(raw)) + except ProtoError as exc: + raise SenderKeyError("Invalid sender-key protobuf") from exc + out: dict[int, list[int | bytes]] = {} + for item in fields: + out.setdefault(item.number, []).append(item.value) + return out + + +def _required_int(fields: dict[int, list[int | bytes]], number: int, name: str) -> int: + values = fields.get(number) or [] + if not values or not isinstance(values[-1], int): + raise SenderKeyError(f"Missing sender-key integer field {name}") + return int(values[-1]) + + +def _required_bytes(fields: dict[int, list[int | bytes]], number: int, name: str) -> bytes: + values = fields.get(number) or [] + if not values or not isinstance(values[-1], bytes): + raise SenderKeyError(f"Missing sender-key bytes field {name}") + return bytes(values[-1]) + + +def _derive(seed: bytes, selector: bytes) -> bytes: + if len(seed) != 32: + raise SenderKeyError("Sender chain key must be 32 bytes") + return hmac.new(seed, selector, hashlib.sha256).digest() + + +def _message_seed(chain_key: bytes) -> bytes: + return _derive(chain_key, b"\x01") + + +def _next_chain_key(chain_key: bytes) -> bytes: + return _derive(chain_key, b"\x02") + + +def _cache_skipped(state: SenderKeyState, iteration: int, seed: bytes) -> None: + state.skipped[int(iteration)] = bytes(seed) + while len(state.skipped) > MAX_SENDER_MESSAGE_KEYS: + oldest = min(state.skipped) + state.skipped.pop(oldest, None) + + +def _message_key_for(state: SenderKeyState, iteration: int) -> SenderMessageKey: + target = int(iteration) + if target < 0: + raise SenderKeyError("Sender-key message iteration must be non-negative") + current = int(state.iteration) + if current > target: + seed = state.skipped.pop(target, None) + if seed is None: + raise SenderKeyError(f"Received sender-key message with old counter {target}; current is {current}") + return SenderMessageKey(target, seed) + if target - current > MAX_SENDER_MESSAGE_KEYS: + raise SenderKeyError("Sender-key message is over 2000 iterations into the future") + + while current < target: + _cache_skipped(state, current, _message_seed(state.chain_key)) + state.chain_key = _next_chain_key(state.chain_key) + current += 1 + state.iteration = current + + seed = _message_seed(state.chain_key) + state.chain_key = _next_chain_key(state.chain_key) + state.iteration = target + 1 + return SenderMessageKey(target, seed) + + +def new_sender_key_state() -> SenderKeyState: + pair = generate_curve_keypair() + return SenderKeyState( + key_id=secrets.randbelow(0x7FFFFFFF), + iteration=0, + chain_key=os.urandom(32), + signing_public=pair.public, + signing_private=pair.private, + ) + + +class EncryptedSenderKeyStore: + def __init__(self, auth: EncryptedAuthStore): + self.auth = auth + + @staticmethod + def name(group_id: str, author_jid: str) -> str: + return f"{group_id}::{author_jid}" + + def load(self, group_id: str, author_jid: str) -> SenderKeyState | None: + value = self.auth.get_json("signal-sender-key", self.name(group_id, author_jid)) + if not value: + return None + + def raw(name: str, *, optional: bool = False) -> bytes | None: + item = value.get(name) + if item in (None, "") and optional: + return None + try: + return base64.b64decode(str(item), validate=True) + except Exception as exc: + raise SenderKeyError(f"Stored sender-key field {name} is invalid") from exc + + skipped: dict[int, bytes] = {} + for item in value.get("skipped") or []: + if not isinstance(item, dict): + continue + try: + skipped[int(item["iteration"])] = base64.b64decode(str(item["seed"]), validate=True) + except Exception as exc: + raise SenderKeyError("Stored skipped sender-key message key is invalid") from exc + + return SenderKeyState( + key_id=int(value["key_id"]), + iteration=int(value["iteration"]), + chain_key=raw("chain_key") or b"", + signing_public=raw("signing_public") or b"", + signing_private=raw("signing_private", optional=True), + skipped=skipped, + ) + + def save(self, group_id: str, author_jid: str, state: SenderKeyState) -> None: + def b64(value: bytes | None) -> str | None: + return base64.b64encode(value).decode("ascii") if value is not None else None + self.auth.put_json( + "signal-sender-key", + self.name(group_id, author_jid), + { + "key_id": state.key_id, + "iteration": state.iteration, + "chain_key": b64(state.chain_key), + "signing_public": b64(state.signing_public), + "signing_private": b64(state.signing_private), + "skipped": [ + {"iteration": iteration, "seed": b64(seed)} + for iteration, seed in sorted(state.skipped.items()) + ], + }, + ) + + def ensure_own(self, group_id: str, author_jid: str) -> SenderKeyState: + state = self.load(group_id, author_jid) + if state is None: + state = new_sender_key_state() + self.save(group_id, author_jid, state) + if state.signing_private is None: + raise SenderKeyError("Stored sender-key state is receive-only and cannot encrypt") + return state + + def distribution(self, group_id: str, author_jid: str) -> bytes: + state = self.ensure_own(group_id, author_jid) + return SenderKeyDistribution( + state.key_id, + state.iteration, + state.chain_key, + state.signing_public, + ).serialize() + + def process_distribution(self, group_id: str, author_jid: str, raw: bytes) -> SenderKeyState: + incoming = SenderKeyDistribution.parse(raw) + state = SenderKeyState( + key_id=incoming.key_id, + iteration=incoming.iteration, + chain_key=incoming.chain_key, + signing_public=incoming.signing_public, + signing_private=None, + ) + self.save(group_id, author_jid, state) + return state + + def encrypt(self, group_id: str, author_jid: str, plaintext: bytes) -> bytes: + state = self.ensure_own(group_id, author_jid) + # Mirror the current libsignal sender-key iteration behavior used by WA Web. + target_iteration = 0 if state.iteration == 0 else state.iteration + 1 + message_key = _message_key_for(state, target_iteration) + ciphertext = aes_cbc_encrypt(message_key.cipher_key, message_key.iv, bytes(plaintext)) + assert state.signing_private is not None + message = SenderKeyMessage.create( + key_id=state.key_id, + iteration=message_key.iteration, + ciphertext=ciphertext, + signing_private=state.signing_private, + ) + self.save(group_id, author_jid, state) + return message.serialize() + + def decrypt(self, group_id: str, author_jid: str, raw: bytes) -> bytes: + message = SenderKeyMessage.parse(raw) + state = self.load(group_id, author_jid) + if state is None or state.key_id != message.key_id: + raise SenderKeyError("No matching sender-key state for group message") + if not message.verify(state.signing_public): + raise SenderKeyError("Invalid sender-key message signature") + message_key = _message_key_for(state, message.iteration) + try: + plaintext = aes_cbc_decrypt(message_key.cipher_key, message_key.iv, message.ciphertext) + except Exception as exc: + raise SenderKeyError("Invalid sender-key ciphertext") from exc + self.save(group_id, author_jid, state) + return plaintext + + +def encode_sender_key_distribution_message(group_id: str, distribution: bytes) -> bytes: + """Encode WAProto.Message.senderKeyDistributionMessage (Message field 2).""" + nested = b"".join((field_bytes(1, str(group_id)), field_bytes(2, bytes(distribution)))) + return field_message(2, [nested]) + + +def decode_sender_key_distribution_message(message: bytes) -> tuple[str, bytes] | None: + """Decode WAProto.Message.senderKeyDistributionMessage, including DeviceSentMessage wrappers.""" + try: + fields = decode_fields(bytes(message)) + except ProtoError as exc: + raise SenderKeyError("Invalid WhatsApp Message protobuf") from exc + candidates = [field.value for field in fields if field.number == 2 and isinstance(field.value, bytes)] + if not candidates: + # WAProto.Message.deviceSentMessage = field 31; DeviceSentMessage.message = field 2. + wrappers = [field.value for field in fields if field.number == 31 and isinstance(field.value, bytes)] + if wrappers: + wrapped = _field_map(bytes(wrappers[-1])) + nested_message = _required_bytes(wrapped, 2, "deviceSentMessage.message") + return decode_sender_key_distribution_message(nested_message) + return None + nested = _field_map(bytes(candidates[-1])) + group_raw = _required_bytes(nested, 1, "groupId") + distribution = _required_bytes(nested, 2, "axolotlSenderKeyDistributionMessage") + try: + group_id = group_raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise SenderKeyError("Sender-key group id is not UTF-8") from exc + return group_id, distribution + + +__all__ = [ + "SENDER_KEY_VERSION", + "SENDER_KEY_VERSION_BYTE", + "SenderKeyError", + "SenderMessageKey", + "SenderKeyState", + "SenderKeyDistribution", + "SenderKeyMessage", + "EncryptedSenderKeyStore", + "new_sender_key_state", + "encode_sender_key_distribution_message", + "decode_sender_key_distribution_message", +] diff --git a/src/postmaster/whatsapp_v990/service.py b/src/postmaster/whatsapp_v990/service.py new file mode 100644 index 00000000..3c20eed9 --- /dev/null +++ b/src/postmaster/whatsapp_v990/service.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +from contextlib import closing +from dataclasses import dataclass +import json +import os +from pathlib import Path +import sqlite3 +import threading +from typing import Any, Mapping, Protocol + +from .jid import parse_jid +from .receipts import receipt_for_event +from .store import EncryptedAuthStore + + +class WhatsAppServiceError(RuntimeError): + pass + + +class WhatsAppAdapter(Protocol): + async def start_pairing(self) -> Mapping[str, Any]: ... + async def reconnect(self) -> Mapping[str, Any]: ... + async def send_text(self, *, jid: str, text: str, reply_to_message_id: str | None, emit_read_receipt: bool) -> Mapping[str, Any]: ... + async def send_media(self, *, jid: str, stored_file_id: str, caption: str, reply_to_message_id: str | None, emit_read_receipt: bool) -> Mapping[str, Any]: ... + async def list_groups(self) -> list[Mapping[str, Any]]: ... + def status(self) -> Mapping[str, Any]: ... + + +class UnavailableAdapter: + """Safe default until the real current-protocol adapter is installed.""" + async def start_pairing(self): raise WhatsAppServiceError("WhatsApp network adapter is not configured") + async def reconnect(self): raise WhatsAppServiceError("WhatsApp network adapter is not configured") + async def send_text(self, **kwargs): raise WhatsAppServiceError("WhatsApp network adapter is not configured") + async def send_media(self, **kwargs): raise WhatsAppServiceError("WhatsApp network adapter is not configured") + async def list_groups(self): raise WhatsAppServiceError("WhatsApp network adapter is not configured") + def status(self): return {"configured": False, "connected": False} + + +class WhatsAppEventStore: + def __init__(self, path: str): + self.path = str(path) + Path(self.path).parent.mkdir(parents=True, exist_ok=True) + self._lock = threading.RLock() + self._init() + + def _connect(self): + conn = sqlite3.connect(self.path, timeout=30) + conn.row_factory = sqlite3.Row + return conn + + def _init(self): + with closing(self._connect()) as conn: + conn.executescript(""" + CREATE TABLE IF NOT EXISTS whatsapp_messages( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_id TEXT, + jid TEXT NOT NULL, + direction TEXT NOT NULL, + kind TEXT NOT NULL, + text TEXT NOT NULL DEFAULT '', + stored_file_id TEXT, + reply_to_message_id TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS idx_wa_messages_jid_id ON whatsapp_messages(jid,id DESC); + CREATE TABLE IF NOT EXISTS whatsapp_receipts( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_id TEXT, + jid TEXT, + receipt_type TEXT NOT NULL, + source TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + """) + conn.commit() + + def record_message(self, *, message_id: str | None, jid: str, direction: str, kind: str, text: str = "", stored_file_id: str | None = None, reply_to_message_id: str | None = None): + with self._lock, closing(self._connect()) as conn: + conn.execute("INSERT INTO whatsapp_messages(message_id,jid,direction,kind,text,stored_file_id,reply_to_message_id) VALUES(?,?,?,?,?,?,?)", + (message_id, jid, direction, kind, text, stored_file_id, reply_to_message_id)) + conn.commit() + + def list_messages(self, *, jid: str | None = None, limit: int = 100) -> list[dict[str, Any]]: + limit = max(1, min(int(limit), 500)) + with self._lock, closing(self._connect()) as conn: + if jid: + rows = conn.execute("SELECT * FROM whatsapp_messages WHERE jid=? ORDER BY id DESC LIMIT ?", (jid, limit)).fetchall() + else: + rows = conn.execute("SELECT * FROM whatsapp_messages ORDER BY id DESC LIMIT ?", (limit,)).fetchall() + return [dict(r) for r in rows] + + def record_receipt(self, *, message_id: str | None, jid: str | None, receipt_type: str, source: str): + with self._lock, closing(self._connect()) as conn: + conn.execute("INSERT INTO whatsapp_receipts(message_id,jid,receipt_type,source) VALUES(?,?,?,?)", (message_id,jid,receipt_type,source)) + conn.commit() + + def list_receipts(self, *, limit: int = 200) -> list[dict[str, Any]]: + limit = max(1, min(int(limit), 1000)) + with self._lock, closing(self._connect()) as conn: + rows = conn.execute("SELECT * FROM whatsapp_receipts ORDER BY id DESC LIMIT ?", (limit,)).fetchall() + return [dict(r) for r in rows] + + +@dataclass(slots=True) +class WhatsAppService: + auth: EncryptedAuthStore + events: WhatsAppEventStore + adapter: WhatsAppAdapter + + @classmethod + def create( + cls, + *, + auth_db: str, + event_db: str, + key_path: str | None = None, + adapter: WhatsAppAdapter | None = None, + file_store: Any | None = None, + file_owner_id: str | None = None, + file_project_id: str | None = None, + ) -> "WhatsAppService": + auth = EncryptedAuthStore(auth_db, key_path=key_path) + events = WhatsAppEventStore(event_db) + if adapter is None: + # CurrentProtocolAdapter is explicit-action only: construction/status never opens + # a socket. Pairing/reconnect happen only through their dedicated MCP/WebGUI actions. + # Incoming transport events are persisted locally without emitting read receipts. + from .adapter import CurrentProtocolAdapter + adapter = CurrentProtocolAdapter( + auth, + on_message=lambda **kwargs: events.record_message(**kwargs), + on_receipt=lambda **kwargs: events.record_receipt(**kwargs), + file_store=file_store, + file_owner_id=file_owner_id or os.getenv("DEFAULT_OWNER_ID", "default"), + file_project_id=file_project_id, + ) + return cls(auth, events, adapter) + + def status(self) -> dict[str, Any]: + network = dict(self.adapter.status()) + pairing = self.auth.get_json("runtime", "pairing") or {} + return { + "ok": True, + "network": network, + "paired": bool((self.auth.get_json("runtime", "identity") or {}).get("jid")), + "pairing_qr_available": bool(pairing.get("qr")), + "auth_store": self.auth.status(), + "read_receipts": {"incoming_visible": True, "local_read_emits": False, "outbound_reply_may_emit": True}, + } + + async def start_pairing(self) -> dict[str, Any]: + result = dict(await self.adapter.start_pairing()) + qr = result.get("qr") + if qr: + self.auth.put_json("runtime", "pairing", {"qr": str(qr), "expires_at": result.get("expires_at")}) + # Explicitly strip any private material the network adapter might accidentally return. + safe = {k:v for k,v in result.items() if k not in {"private_key","identity_private_key","noise_private_key","secret"}} + safe["private_material_exposed"] = False + return safe + + async def reconnect(self) -> dict[str, Any]: + return dict(await self.adapter.reconnect()) + + def pairing_qr(self) -> str | None: + state = self.auth.get_json("runtime", "pairing") or {} + return str(state.get("qr")) if state.get("qr") else None + + def list_messages(self, *, jid: str | None = None, limit: int = 100) -> dict[str, Any]: + if jid: jid = str(parse_jid(jid)) + return {"ok": True, "messages": self.events.list_messages(jid=jid, limit=limit), "read_receipt_emitted": False} + + async def send_text(self, *, jid: str, text: str, reply_to_message_id: str | None = None) -> dict[str, Any]: + dest = str(parse_jid(jid)) + if not str(text): raise WhatsAppServiceError("WhatsApp text cannot be empty") + decision = receipt_for_event(event="reply" if reply_to_message_id else "send", outbound_action=True, reply_to_message=bool(reply_to_message_id)) + result = dict(await self.adapter.send_text(jid=dest, text=str(text), reply_to_message_id=reply_to_message_id, emit_read_receipt=decision.emit)) + if result.get("ok", True): + self.events.record_message(message_id=result.get("message_id"), jid=dest, direction="out", kind="text", text=str(text), reply_to_message_id=reply_to_message_id) + if decision.emit: + self.events.record_receipt(message_id=reply_to_message_id, jid=dest, receipt_type="read", source="outbound_reply") + result["read_receipt_emitted"] = decision.emit + result["read_receipt_reason"] = decision.reason + return result + + async def send_media(self, *, jid: str, stored_file_id: str, caption: str = "", reply_to_message_id: str | None = None) -> dict[str, Any]: + dest = str(parse_jid(jid)) + fid = str(stored_file_id or "").strip() + if not fid: raise WhatsAppServiceError("stored_file_id is required; use Postmaster Stored File storage for WhatsApp media") + decision = receipt_for_event(event="reply" if reply_to_message_id else "send", outbound_action=True, reply_to_message=bool(reply_to_message_id)) + result = dict(await self.adapter.send_media(jid=dest, stored_file_id=fid, caption=str(caption or ""), reply_to_message_id=reply_to_message_id, emit_read_receipt=decision.emit)) + if result.get("ok", True): + self.events.record_message(message_id=result.get("message_id"), jid=dest, direction="out", kind="media", text=str(caption or ""), stored_file_id=fid, reply_to_message_id=reply_to_message_id) + if decision.emit: + self.events.record_receipt(message_id=reply_to_message_id, jid=dest, receipt_type="read", source="outbound_reply") + result["read_receipt_emitted"] = decision.emit + return result + + async def list_groups(self) -> dict[str, Any]: + return {"ok": True, "groups": [dict(x) for x in await self.adapter.list_groups()]} + + def record_incoming_message(self, *, message_id: str | None, jid: str, text: str = "", kind: str = "text", stored_file_id: str | None = None): + self.events.record_message(message_id=message_id, jid=str(parse_jid(jid)), direction="in", kind=kind, text=text, stored_file_id=stored_file_id) + + def record_incoming_receipt(self, *, message_id: str | None, jid: str | None, receipt_type: str): + self.events.record_receipt(message_id=message_id, jid=str(parse_jid(jid)) if jid else None, receipt_type=str(receipt_type), source="remote") + + def list_receipts(self, *, limit: int = 200) -> dict[str, Any]: + return {"ok": True, "receipts": self.events.list_receipts(limit=limit), "local_read_emits_receipt": False} diff --git a/src/postmaster/whatsapp_v990/signal_keys.py b/src/postmaster/whatsapp_v990/signal_keys.py new file mode 100644 index 00000000..90e1c44f --- /dev/null +++ b/src/postmaster/whatsapp_v990/signal_keys.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +"""Clean-room Signal-v3 key bundle and X3DH-compatible derivation helpers. + +These helpers match the Curve25519/XEdDSA bundle shape used by WhatsApp's current +libsignal path. They are protocol primitives, not a claim of live server interoperability. +""" + +from dataclasses import dataclass +import hmac +import hashlib +import os + +from .crypto import ( + CurveKeyPair, + curve_shared_key, + generate_curve_keypair, + hkdf_sha256, + signal_public_key, + xeddsa_sign, + xeddsa_verify, +) + + +class SignalKeyError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class SignedPreKey: + key_id: int + key_pair: CurveKeyPair + signature: bytes + + +@dataclass(frozen=True, slots=True) +class SignalPreKeyBundle: + registration_id: int + identity_key: bytes + signed_pre_key_id: int + signed_pre_key: bytes + signed_pre_key_signature: bytes + pre_key_id: int | None = None + pre_key: bytes | None = None + + +def generate_registration_id(random2: bytes | None = None) -> int: + raw = bytes(random2 if random2 is not None else os.urandom(2)) + if len(raw) != 2: + raise SignalKeyError("Signal registration-id entropy must be exactly two bytes") + return int.from_bytes(raw, "little") & 0x3FFF + + +def generate_signed_pre_key(identity: CurveKeyPair, key_id: int, *, pre_key: CurveKeyPair | None = None, random64: bytes | None = None) -> SignedPreKey: + if not 0 <= int(key_id) <= 0xFFFFFF: + raise SignalKeyError("WhatsApp signed-prekey id must fit 24 bits") + pair = pre_key or generate_curve_keypair() + signature = xeddsa_sign(identity.private, signal_public_key(pair.public), random64=random64) + return SignedPreKey(key_id=int(key_id), key_pair=pair, signature=signature) + + +def verify_signed_pre_key(identity_public: bytes, signed_pre_key_public: bytes, signature: bytes) -> bool: + return xeddsa_verify(identity_public, signal_public_key(signed_pre_key_public), signature) + + +def derive_x3dh_initiator( + *, + our_identity_private: bytes, + our_base_private: bytes, + their_identity_public: bytes, + their_signed_pre_key_public: bytes, + their_one_time_pre_key_public: bytes | None = None, +) -> tuple[bytes, bytes]: + """Derive the legacy Signal-v3 initial root/chain pair for the initiator.""" + pieces = [ + b"\xFF" * 32, + curve_shared_key(our_identity_private, _raw_pub(their_signed_pre_key_public)), + curve_shared_key(our_base_private, _raw_pub(their_identity_public)), + curve_shared_key(our_base_private, _raw_pub(their_signed_pre_key_public)), + ] + if their_one_time_pre_key_public is not None: + pieces.append(curve_shared_key(our_base_private, _raw_pub(their_one_time_pre_key_public))) + material = hkdf_sha256(b"".join(pieces), 64, salt=None, info=b"WhisperText") + return material[:32], material[32:] + + +def derive_x3dh_responder( + *, + our_identity_private: bytes, + our_signed_pre_key_private: bytes, + their_identity_public: bytes, + their_base_public: bytes, + our_one_time_pre_key_private: bytes | None = None, +) -> tuple[bytes, bytes]: + """Responder mirror of :func:`derive_x3dh_initiator`.""" + pieces = [ + b"\xFF" * 32, + curve_shared_key(our_signed_pre_key_private, _raw_pub(their_identity_public)), + curve_shared_key(our_identity_private, _raw_pub(their_base_public)), + curve_shared_key(our_signed_pre_key_private, _raw_pub(their_base_public)), + ] + if our_one_time_pre_key_private is not None: + pieces.append(curve_shared_key(our_one_time_pre_key_private, _raw_pub(their_base_public))) + material = hkdf_sha256(b"".join(pieces), 64, salt=None, info=b"WhisperText") + return material[:32], material[32:] + + +def chain_message_seed(chain_key: bytes) -> bytes: + if len(chain_key) != 32: + raise SignalKeyError("Signal chain key must be 32 bytes") + return hmac.new(bytes(chain_key), b"\x01", hashlib.sha256).digest() + + +def next_chain_key(chain_key: bytes) -> bytes: + if len(chain_key) != 32: + raise SignalKeyError("Signal chain key must be 32 bytes") + return hmac.new(bytes(chain_key), b"\x02", hashlib.sha256).digest() + + +def derive_message_keys(chain_key: bytes) -> tuple[bytes, bytes, bytes]: + seed = chain_message_seed(chain_key) + material = hkdf_sha256(seed, 80, salt=None, info=b"WhisperMessageKeys") + return material[:32], material[32:64], material[64:80] + + +def _raw_pub(public: bytes) -> bytes: + raw = bytes(public) + if len(raw) == 33 and raw[0] == 5: + raw = raw[1:] + if len(raw) != 32: + raise SignalKeyError("Signal Curve25519 public key must be 32 bytes or 0x05-prefixed 33 bytes") + return raw diff --git a/src/postmaster/whatsapp_v990/signal_server.py b/src/postmaster/whatsapp_v990/signal_server.py new file mode 100644 index 00000000..715f23eb --- /dev/null +++ b/src/postmaster/whatsapp_v990/signal_server.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +"""WhatsApp Signal key-bundle XMPP/WABinary helpers. + +All functions here are pure protocol transforms: no network I/O and no account side effects. +""" + +from dataclasses import dataclass +from typing import Iterable, Mapping + +from .binary import BinaryNode +from .crypto import CurveKeyPair +from .signal_keys import SignalPreKeyBundle, SignedPreKey + +KEY_BUNDLE_TYPE = b"\x05" + + +class SignalServerError(ValueError): + pass + + +def encode_uint_be(value: int, width: int | None = None) -> bytes: + value = int(value) + if value < 0: + raise SignalServerError("Signal integer must be non-negative") + if width is None: + width = max(1, (value.bit_length() + 7) // 8) + if width <= 0 or value >= (1 << (width * 8)): + raise SignalServerError("Signal integer does not fit requested width") + return value.to_bytes(width, "big") + + +def decode_uint_be(raw: bytes, *, width: int | None = None) -> int: + value = bytes(raw) + if width is not None and len(value) != width: + raise SignalServerError(f"Signal integer must be {width} bytes") + if not value: + raise SignalServerError("Signal integer cannot be empty") + return int.from_bytes(value, "big") + + +def _bytes_content(node: BinaryNode | None, label: str) -> bytes: + if node is None or not isinstance(node.content, bytes): + raise SignalServerError(f"Missing Signal {label}") + return bytes(node.content) + + +def xmpp_pre_key(key_id: int, pair: CurveKeyPair) -> BinaryNode: + return BinaryNode("key", {}, [ + BinaryNode("id", {}, encode_uint_be(key_id, 3)), + BinaryNode("value", {}, pair.public), + ]) + + +def xmpp_signed_pre_key(value: SignedPreKey) -> BinaryNode: + return BinaryNode("skey", {}, [ + BinaryNode("id", {}, encode_uint_be(value.key_id, 3)), + BinaryNode("value", {}, value.key_pair.public), + BinaryNode("signature", {}, value.signature), + ]) + + +def build_prekey_count_query(*, stanza_id: str | None = None) -> BinaryNode: + attrs = {"xmlns": "encrypt", "type": "get", "to": "s.whatsapp.net"} + if stanza_id: + attrs["id"] = stanza_id + return BinaryNode("iq", attrs, [BinaryNode("count")]) + + +def parse_prekey_count(response: BinaryNode) -> int: + count = response.child("count") + if count is None: + raise SignalServerError("Signal pre-key count response has no count node") + try: + value = int(count.attrs.get("value", "")) + except ValueError as exc: + raise SignalServerError("Signal pre-key count is not an integer") from exc + if value < 0: + raise SignalServerError("Signal pre-key count cannot be negative") + return value + + +def build_prekey_upload( + *, + registration_id: int, + identity_public: bytes, + signed_pre_key: SignedPreKey, + pre_keys: Mapping[int, CurveKeyPair], + stanza_id: str | None = None, +) -> BinaryNode: + identity = bytes(identity_public) + if len(identity) != 32: + raise SignalServerError("Signal identity public key must be 32 bytes") + attrs = {"xmlns": "encrypt", "type": "set", "to": "s.whatsapp.net"} + if stanza_id: + attrs["id"] = stanza_id + ordered = [xmpp_pre_key(key_id, pre_keys[key_id]) for key_id in sorted(pre_keys)] + return BinaryNode("iq", attrs, [ + BinaryNode("registration", {}, encode_uint_be(registration_id)), + BinaryNode("type", {}, KEY_BUNDLE_TYPE), + BinaryNode("identity", {}, identity), + BinaryNode("list", {}, ordered), + xmpp_signed_pre_key(signed_pre_key), + ]) + + +def build_session_query(jids: Iterable[str], *, force_identity: bool = False, stanza_id: str | None = None) -> BinaryNode: + users = [] + seen = set() + for jid in jids: + value = str(jid or "").strip() + if not value or value in seen: + continue + seen.add(value) + attrs = {"jid": value} + if force_identity: + attrs["reason"] = "identity" + users.append(BinaryNode("user", attrs)) + if not users: + raise SignalServerError("At least one JID is required for Signal session fetch") + attrs = {"xmlns": "encrypt", "type": "get", "to": "s.whatsapp.net"} + if stanza_id: + attrs["id"] = stanza_id + return BinaryNode("iq", attrs, [BinaryNode("key", {}, users)]) + + +def _extract_key(node: BinaryNode | None, *, signed: bool) -> tuple[int, bytes, bytes | None]: + if node is None: + raise SignalServerError("Missing Signal key node") + key_id = decode_uint_be(_bytes_content(node.child("id"), "key id"), width=3) + public = _bytes_content(node.child("value"), "key value") + if len(public) != 32: + raise SignalServerError("Signal pre-key public key must be 32 bytes") + signature = _bytes_content(node.child("signature"), "signed pre-key signature") if signed else None + if signature is not None and len(signature) != 64: + raise SignalServerError("Signal signed pre-key signature must be 64 bytes") + return key_id, public, signature + + +def parse_session_bundles(response: BinaryNode) -> dict[str, SignalPreKeyBundle]: + listing = response.child("list") + if listing is None: + raise SignalServerError("Signal session response has no list node") + bundles: dict[str, SignalPreKeyBundle] = {} + for user in listing.children("user"): + jid = str(user.attrs.get("jid") or "").strip() + if not jid: + raise SignalServerError("Signal session user has no JID") + error = user.child("error") + if error is not None: + continue + registration = decode_uint_be(_bytes_content(user.child("registration"), "registration")) + type_node = _bytes_content(user.child("type"), "key bundle type") + if type_node != KEY_BUNDLE_TYPE: + raise SignalServerError("Unsupported Signal key bundle type") + identity = _bytes_content(user.child("identity"), "identity") + if len(identity) != 32: + raise SignalServerError("Signal identity key must be 32 bytes") + signed_id, signed_public, signature = _extract_key(user.child("skey"), signed=True) + pre_node = user.child("key") + pre_id: int | None = None + pre_public: bytes | None = None + if pre_node is not None: + pre_id, pre_public, _ = _extract_key(pre_node, signed=False) + bundles[jid] = SignalPreKeyBundle( + registration_id=registration, + identity_key=identity, + signed_pre_key_id=signed_id, + signed_pre_key=signed_public, + signed_pre_key_signature=signature or b"", + pre_key_id=pre_id, + pre_key=pre_public, + ) + return bundles + + +__all__ = [ + "KEY_BUNDLE_TYPE", "SignalServerError", "encode_uint_be", "decode_uint_be", + "xmpp_pre_key", "xmpp_signed_pre_key", "build_prekey_count_query", "parse_prekey_count", + "build_prekey_upload", "build_session_query", "parse_session_bundles", +] diff --git a/src/postmaster/whatsapp_v990/signal_session.py b/src/postmaster/whatsapp_v990/signal_session.py new file mode 100644 index 00000000..ec80fded --- /dev/null +++ b/src/postmaster/whatsapp_v990/signal_session.py @@ -0,0 +1,403 @@ +from __future__ import annotations + +"""Clean-room legacy Signal-v3 Double Ratchet session used by WhatsApp Web. + +This implements the X3DH + Whisper v3 session semantics needed by the current WhatsApp +wire envelope. State is serializable for encrypted-at-rest persistence; network/device +discovery is intentionally outside this module. +""" + +from dataclasses import dataclass, field +import base64 +import hashlib +import hmac +from typing import Any, Mapping + +from .crypto import CurveKeyPair, aes_cbc_decrypt, aes_cbc_encrypt, curve_shared_key, generate_curve_keypair, hkdf_sha256 +from .signal_keys import ( + SignalPreKeyBundle, + SignedPreKey, + derive_x3dh_initiator, + derive_x3dh_responder, + verify_signed_pre_key, +) +from .signal_wire import PreKeyWhisperMessageV3, SignalMessageV3 + + +class SignalSessionError(ValueError): + pass + + +MAX_FUTURE_MESSAGES = 2000 +MAX_STORED_MESSAGE_KEYS = 2000 + + +def _raw_public(value: bytes) -> bytes: + raw = bytes(value) + if len(raw) == 33 and raw[0] == 5: + raw = raw[1:] + if len(raw) != 32: + raise SignalSessionError("Signal Curve25519 public key must be 32 bytes or 0x05-prefixed 33 bytes") + return raw + + +def _b64(raw: bytes) -> str: + return base64.b64encode(bytes(raw)).decode("ascii") + + +def _unb64(value: Any, *, length: int | None = None) -> bytes: + try: + raw = base64.b64decode(str(value), validate=True) + except Exception as exc: + raise SignalSessionError("Invalid Base64 in Signal session state") from exc + if length is not None and len(raw) != length: + raise SignalSessionError(f"Signal session value must be {length} bytes") + return raw + + +def _root_chain(root_key: bytes, our_private: bytes, their_public: bytes) -> tuple[bytes, bytes]: + if len(root_key) != 32: + raise SignalSessionError("Signal root key must be 32 bytes") + shared = curve_shared_key(our_private, _raw_public(their_public)) + material = hkdf_sha256(shared, 64, salt=root_key, info=b"WhisperRatchet") + return material[:32], material[32:] + + +def _message_seed(chain_key: bytes) -> bytes: + return hmac.new(bytes(chain_key), b"\x01", hashlib.sha256).digest() + + +def _next_chain_key(chain_key: bytes) -> bytes: + return hmac.new(bytes(chain_key), b"\x02", hashlib.sha256).digest() + + +def _message_keys_from_seed(seed: bytes) -> tuple[bytes, bytes, bytes]: + if len(seed) != 32: + raise SignalSessionError("Signal message seed must be 32 bytes") + material = hkdf_sha256(seed, 80, salt=bytes(32), info=b"WhisperMessageKeys") + return material[:32], material[32:64], material[64:80] + + +@dataclass(slots=True) +class ChainState: + key: bytes | None + counter: int = -1 + skipped: dict[int, bytes] = field(default_factory=dict) + sending: bool = False + + def fill_to(self, target: int) -> None: + if target < 0: + raise SignalSessionError("Signal message counter cannot be negative") + if self.counter >= target: + return + if self.key is None: + raise SignalSessionError("Signal chain is closed") + if target - self.counter > MAX_FUTURE_MESSAGES: + raise SignalSessionError("Signal message is over 2000 counters into the future") + while self.counter < target: + assert self.key is not None + next_counter = self.counter + 1 + self.skipped[next_counter] = _message_seed(self.key) + self.key = _next_chain_key(self.key) + self.counter = next_counter + if len(self.skipped) > MAX_STORED_MESSAGE_KEYS: + oldest = min(self.skipped) + del self.skipped[oldest] + + def take_seed(self, counter: int) -> bytes: + self.fill_to(counter) + try: + return self.skipped.pop(counter) + except KeyError as exc: + raise SignalSessionError("Signal message key was already used or never derived") from exc + + def close(self) -> None: + self.key = None + + def to_json(self) -> dict[str, Any]: + return { + "key": _b64(self.key) if self.key is not None else None, + "counter": self.counter, + "skipped": {str(k): _b64(v) for k, v in self.skipped.items()}, + "sending": self.sending, + } + + @classmethod + def from_json(cls, value: Mapping[str, Any]) -> "ChainState": + key = _unb64(value["key"], length=32) if value.get("key") else None + skipped = {int(k): _unb64(v, length=32) for k, v in dict(value.get("skipped") or {}).items()} + return cls(key=key, counter=int(value.get("counter", -1)), skipped=skipped, sending=bool(value.get("sending"))) + + +@dataclass(slots=True) +class PendingPreKey: + base_key: bytes + signed_pre_key_id: int + pre_key_id: int | None = None + + def to_json(self) -> dict[str, Any]: + return {"base_key": _b64(self.base_key), "signed_pre_key_id": self.signed_pre_key_id, "pre_key_id": self.pre_key_id} + + @classmethod + def from_json(cls, value: Mapping[str, Any]) -> "PendingPreKey": + return cls(_unb64(value["base_key"], length=32), int(value["signed_pre_key_id"]), int(value["pre_key_id"]) if value.get("pre_key_id") is not None else None) + + +@dataclass(slots=True) +class SignalSession: + registration_id: int + local_registration_id: int + local_identity: CurveKeyPair + remote_identity: bytes + root_key: bytes + ratchet_key: CurveKeyPair + last_remote_ratchet: bytes + previous_counter: int = 0 + chains: dict[str, ChainState] = field(default_factory=dict) + pending_pre_key: PendingPreKey | None = None + base_key: bytes = b"" + + @staticmethod + def _chain_id(public: bytes) -> str: + return _raw_public(public).hex() + + def chain(self, public: bytes) -> ChainState | None: + return self.chains.get(self._chain_id(public)) + + def set_chain(self, public: bytes, chain: ChainState) -> None: + self.chains[self._chain_id(public)] = chain + + def delete_chain(self, public: bytes) -> None: + self.chains.pop(self._chain_id(public), None) + + def _create_chain(self, remote_public: bytes, *, sending: bool) -> None: + root, chain_key = _root_chain(self.root_key, self.ratchet_key.private, remote_public) + self.root_key = root + key_id = self.ratchet_key.public if sending else _raw_public(remote_public) + self.set_chain(key_id, ChainState(chain_key, counter=-1, sending=sending)) + + def maybe_step_ratchet(self, remote_public: bytes, previous_counter: int) -> None: + remote = _raw_public(remote_public) + if self.chain(remote) is not None: + return + previous = self.chain(self.last_remote_ratchet) + if previous is not None: + previous.fill_to(previous_counter) + previous.close() + + # Receiving chain: DH(current local ratchet, new remote ratchet). + self._create_chain(remote, sending=False) + + old_sending = self.chain(self.ratchet_key.public) + if old_sending is not None: + self.previous_counter = old_sending.counter + self.delete_chain(self.ratchet_key.public) + + # Rotate our ratchet and derive the next sending chain from the new remote key. + self.ratchet_key = generate_curve_keypair() + self._create_chain(remote, sending=True) + self.last_remote_ratchet = remote + + def encrypt(self, plaintext: bytes) -> tuple[str, bytes]: + chain = self.chain(self.ratchet_key.public) + if chain is None or not chain.sending: + raise SignalSessionError("Signal session has no sending chain") + counter = chain.counter + 1 + seed = chain.take_seed(counter) + cipher_key, mac_key, iv = _message_keys_from_seed(seed) + message = SignalMessageV3( + ratchet_key=self.ratchet_key.public, + counter=counter, + previous_counter=max(0, self.previous_counter), + ciphertext=aes_cbc_encrypt(cipher_key, iv, bytes(plaintext)), + ) + serialized = message.serialize( + mac_key=mac_key, + sender_identity_key=self.local_identity.public, + receiver_identity_key=self.remote_identity, + ) + if self.pending_pre_key is None: + return "msg", serialized + pending = self.pending_pre_key + wrapped = PreKeyWhisperMessageV3( + registration_id=self.local_registration_id, + pre_key_id=pending.pre_key_id, + signed_pre_key_id=pending.signed_pre_key_id, + base_key=pending.base_key, + identity_key=self.local_identity.public, + message=serialized, + ).serialize() + return "pkmsg", wrapped + + def decrypt_signal(self, raw: bytes) -> bytes: + message = SignalMessageV3.parse(raw) + self.maybe_step_ratchet(message.ratchet_key, message.previous_counter) + chain = self.chain(message.ratchet_key) + if chain is None or chain.sending: + raise SignalSessionError("Signal session has no receiving chain for message ratchet key") + seed = chain.take_seed(message.counter) + cipher_key, mac_key, iv = _message_keys_from_seed(seed) + if not message.verify_mac( + mac_key=mac_key, + sender_identity_key=self.remote_identity, + receiver_identity_key=self.local_identity.public, + ): + raise SignalSessionError("Signal message MAC mismatch") + try: + plaintext = aes_cbc_decrypt(cipher_key, iv, message.ciphertext) + except Exception as exc: + raise SignalSessionError("Signal message decryption failed") from exc + self.pending_pre_key = None + return plaintext + + def to_json(self) -> dict[str, Any]: + return { + "registration_id": self.registration_id, + "local_registration_id": self.local_registration_id, + "local_identity_private": _b64(self.local_identity.private), + "local_identity_public": _b64(self.local_identity.public), + "remote_identity": _b64(_raw_public(self.remote_identity)), + "root_key": _b64(self.root_key), + "ratchet_private": _b64(self.ratchet_key.private), + "ratchet_public": _b64(self.ratchet_key.public), + "last_remote_ratchet": _b64(_raw_public(self.last_remote_ratchet)), + "previous_counter": self.previous_counter, + "chains": {k: v.to_json() for k, v in self.chains.items()}, + "pending_pre_key": self.pending_pre_key.to_json() if self.pending_pre_key else None, + "base_key": _b64(_raw_public(self.base_key)) if self.base_key else "", + } + + @classmethod + def from_json(cls, value: Mapping[str, Any]) -> "SignalSession": + local = CurveKeyPair(_unb64(value["local_identity_private"], length=32), _unb64(value["local_identity_public"], length=32)) + ratchet = CurveKeyPair(_unb64(value["ratchet_private"], length=32), _unb64(value["ratchet_public"], length=32)) + pending_raw = value.get("pending_pre_key") + return cls( + registration_id=int(value["registration_id"]), + local_registration_id=int(value.get("local_registration_id", 0)), + local_identity=local, + remote_identity=_unb64(value["remote_identity"], length=32), + root_key=_unb64(value["root_key"], length=32), + ratchet_key=ratchet, + last_remote_ratchet=_unb64(value["last_remote_ratchet"], length=32), + previous_counter=int(value.get("previous_counter", 0)), + chains={str(k): ChainState.from_json(v) for k, v in dict(value.get("chains") or {}).items()}, + pending_pre_key=PendingPreKey.from_json(pending_raw) if isinstance(pending_raw, Mapping) else None, + base_key=_unb64(value["base_key"], length=32) if value.get("base_key") else b"", + ) + + +def initialize_outgoing_session( + *, + our_identity: CurveKeyPair, + bundle: SignalPreKeyBundle, + our_registration_id: int, + base_key: CurveKeyPair | None = None, + ratchet_key: CurveKeyPair | None = None, +) -> SignalSession: + if not verify_signed_pre_key(bundle.identity_key, bundle.signed_pre_key, bundle.signed_pre_key_signature): + raise SignalSessionError("Remote Signal signed pre-key signature is invalid") + base = base_key or generate_curve_keypair() + root, _initial_chain = derive_x3dh_initiator( + our_identity_private=our_identity.private, + our_base_private=base.private, + their_identity_public=bundle.identity_key, + their_signed_pre_key_public=bundle.signed_pre_key, + their_one_time_pre_key_public=bundle.pre_key, + ) + local_ratchet = ratchet_key or generate_curve_keypair() + session = SignalSession( + registration_id=int(bundle.registration_id), + local_registration_id=int(our_registration_id), + local_identity=our_identity, + remote_identity=_raw_public(bundle.identity_key), + root_key=root, + ratchet_key=local_ratchet, + last_remote_ratchet=_raw_public(bundle.signed_pre_key), + base_key=base.public, + pending_pre_key=PendingPreKey(base.public, int(bundle.signed_pre_key_id), bundle.pre_key_id), + ) + session._create_chain(bundle.signed_pre_key, sending=True) + return session + + +def initialize_incoming_session( + *, + our_identity: CurveKeyPair, + our_signed_pre_key: SignedPreKey, + message: PreKeyWhisperMessageV3, + our_one_time_pre_key: CurveKeyPair | None = None, + our_registration_id: int = 0, +) -> SignalSession: + if message.signed_pre_key_id != our_signed_pre_key.key_id: + raise SignalSessionError("Incoming Signal pre-key message references an unknown signed pre-key") + if message.pre_key_id is not None and our_one_time_pre_key is None: + raise SignalSessionError("Incoming Signal pre-key message requires a one-time pre-key") + root, _initial_chain = derive_x3dh_responder( + our_identity_private=our_identity.private, + our_signed_pre_key_private=our_signed_pre_key.key_pair.private, + their_identity_public=message.identity_key, + their_base_public=message.base_key, + our_one_time_pre_key_private=our_one_time_pre_key.private if our_one_time_pre_key else None, + ) + return SignalSession( + registration_id=int(message.registration_id), + local_registration_id=int(our_registration_id), + local_identity=our_identity, + remote_identity=_raw_public(message.identity_key), + root_key=root, + ratchet_key=our_signed_pre_key.key_pair, + last_remote_ratchet=_raw_public(message.base_key), + base_key=_raw_public(message.base_key), + ) + + +def decrypt_prekey_message( + raw: bytes, + *, + our_identity: CurveKeyPair, + our_signed_pre_key: SignedPreKey, + our_one_time_pre_key: CurveKeyPair | None = None, + our_registration_id: int = 0, +) -> tuple[SignalSession, bytes, int | None]: + envelope = PreKeyWhisperMessageV3.parse(raw) + session = initialize_incoming_session( + our_identity=our_identity, + our_signed_pre_key=our_signed_pre_key, + message=envelope, + our_one_time_pre_key=our_one_time_pre_key, + our_registration_id=our_registration_id, + ) + plaintext = session.decrypt_signal(envelope.message) + return session, plaintext, envelope.pre_key_id + + +class EncryptedSignalSessionStore: + """Persist Signal session JSON inside EncryptedAuthStore without exposing private material.""" + + def __init__(self, encrypted_store: Any): + self.store = encrypted_store + + @staticmethod + def _name(address: str) -> str: + clean = str(address or "").strip() + if not clean: + raise SignalSessionError("Signal session address is required") + return clean + + def load(self, address: str) -> SignalSession | None: + raw = self.store.get_json("signal-session", self._name(address)) + return SignalSession.from_json(raw) if raw else None + + def save(self, address: str, session: SignalSession) -> None: + self.store.put_json("signal-session", self._name(address), session.to_json()) + + def delete(self, address: str) -> bool: + return bool(self.store.delete("signal-session", self._name(address))) + + +__all__ = [ + "SignalSessionError", "ChainState", "PendingPreKey", "SignalSession", + "initialize_outgoing_session", "initialize_incoming_session", "decrypt_prekey_message", + "EncryptedSignalSessionStore", "MAX_FUTURE_MESSAGES", +] diff --git a/src/postmaster/whatsapp_v990/signal_wire.py b/src/postmaster/whatsapp_v990/signal_wire.py new file mode 100644 index 00000000..95a06aa4 --- /dev/null +++ b/src/postmaster/whatsapp_v990/signal_wire.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +"""Clean-room Signal v3 wire messages used by the current WhatsApp libsignal path. + +This module deliberately implements only the stable, observable wire envelope used by +WhatsApp's current ``libsignal`` repository: Signal/Whisper messages and +PreKeyWhisper messages. It does not claim to be a full Signal session implementation. +""" + +from dataclasses import dataclass +import hashlib +import hmac + +from .crypto import signal_public_key +from .proto import ProtoError, decode_fields, field_bytes, field_varint + + +SIGNAL_V3 = 3 +SIGNAL_MAC_LENGTH = 8 + + +class SignalWireError(ValueError): + pass + + +def _version_byte(message_version: int = SIGNAL_V3, current_version: int = SIGNAL_V3) -> int: + if not 0 <= int(message_version) <= 15 or not 0 <= int(current_version) <= 15: + raise SignalWireError("Signal version nibbles must fit four bits") + return ((int(message_version) & 0xF) << 4) | (int(current_version) & 0xF) + + +def _parse_version(value: int, *, current_version: int = SIGNAL_V3) -> int: + high, low = (value >> 4) & 0xF, value & 0xF + if low != current_version: + raise SignalWireError(f"Unexpected Signal current-version nibble {low}") + if high < SIGNAL_V3: + raise SignalWireError(f"Legacy Signal message version {high}") + if high > current_version: + raise SignalWireError(f"Unsupported Signal message version {high}") + return high + + +def _field_map(data: bytes) -> dict[int, list[int | bytes]]: + try: + fields = decode_fields(data) + except ProtoError as exc: + raise SignalWireError("Invalid Signal protobuf encoding") from exc + out: dict[int, list[int | bytes]] = {} + for field in fields: + out.setdefault(field.number, []).append(field.value) + return out + + +def _required_bytes(fields: dict[int, list[int | bytes]], number: int, name: str) -> bytes: + values = fields.get(number) + if not values or not isinstance(values[-1], bytes): + raise SignalWireError(f"Missing Signal field {name}") + return bytes(values[-1]) + + +def _required_int(fields: dict[int, list[int | bytes]], number: int, name: str) -> int: + values = fields.get(number) + if not values or not isinstance(values[-1], int): + raise SignalWireError(f"Missing Signal field {name}") + value = int(values[-1]) + if not 0 <= value <= 0xFFFFFFFF: + raise SignalWireError(f"Signal field {name} is outside uint32 range") + return value + + +def _optional_int(fields: dict[int, list[int | bytes]], number: int) -> int | None: + values = fields.get(number) + if not values: + return None + if not isinstance(values[-1], int): + raise SignalWireError(f"Signal field {number} must be varint") + value = int(values[-1]) + if not 0 <= value <= 0xFFFFFFFF: + raise SignalWireError(f"Signal field {number} is outside uint32 range") + return value + + +def signal_message_mac( + *, + mac_key: bytes, + sender_identity_key: bytes, + receiver_identity_key: bytes, + serialized_without_mac: bytes, +) -> bytes: + if len(mac_key) != 32: + raise SignalWireError("Signal MAC key must be 32 bytes") + sender = signal_public_key(sender_identity_key) + receiver = signal_public_key(receiver_identity_key) + return hmac.new(bytes(mac_key), sender + receiver + bytes(serialized_without_mac), hashlib.sha256).digest()[:SIGNAL_MAC_LENGTH] + + +@dataclass(frozen=True, slots=True) +class SignalMessageV3: + ratchet_key: bytes + counter: int + previous_counter: int + ciphertext: bytes + mac: bytes = b"" + message_version: int = SIGNAL_V3 + + def protobuf(self) -> bytes: + ratchet = signal_public_key(self.ratchet_key) + return b"".join( + ( + field_bytes(1, ratchet), + field_varint(2, self.counter), + field_varint(3, self.previous_counter), + field_bytes(4, self.ciphertext), + ) + ) + + def unsigned_bytes(self) -> bytes: + return bytes((_version_byte(self.message_version),)) + self.protobuf() + + def serialize( + self, + *, + mac_key: bytes, + sender_identity_key: bytes, + receiver_identity_key: bytes, + ) -> bytes: + unsigned = self.unsigned_bytes() + mac = signal_message_mac( + mac_key=mac_key, + sender_identity_key=sender_identity_key, + receiver_identity_key=receiver_identity_key, + serialized_without_mac=unsigned, + ) + return unsigned + mac + + @classmethod + def parse(cls, raw: bytes) -> "SignalMessageV3": + data = bytes(raw) + if len(data) < 1 + SIGNAL_MAC_LENGTH: + raise SignalWireError("Signal message is too short") + version = _parse_version(data[0]) + proto = data[1:-SIGNAL_MAC_LENGTH] + mac = data[-SIGNAL_MAC_LENGTH:] + fields = _field_map(proto) + ratchet = _required_bytes(fields, 1, "ratchetKey") + if len(ratchet) != 33 or ratchet[0] != 5: + raise SignalWireError("Signal ratchet key must use 0x05-prefixed Curve25519 encoding") + return cls( + ratchet_key=ratchet, + counter=_required_int(fields, 2, "counter"), + previous_counter=_optional_int(fields, 3) or 0, + ciphertext=_required_bytes(fields, 4, "ciphertext"), + mac=mac, + message_version=version, + ) + + def verify_mac(self, *, mac_key: bytes, sender_identity_key: bytes, receiver_identity_key: bytes) -> bool: + expected = signal_message_mac( + mac_key=mac_key, + sender_identity_key=sender_identity_key, + receiver_identity_key=receiver_identity_key, + serialized_without_mac=self.unsigned_bytes(), + ) + return hmac.compare_digest(self.mac, expected) + + +@dataclass(frozen=True, slots=True) +class PreKeyWhisperMessageV3: + registration_id: int + pre_key_id: int | None + signed_pre_key_id: int + base_key: bytes + identity_key: bytes + message: bytes + message_version: int = SIGNAL_V3 + + def protobuf(self) -> bytes: + parts: list[bytes] = [] + if self.pre_key_id is not None: + parts.append(field_varint(1, self.pre_key_id)) + parts.extend( + ( + field_bytes(2, signal_public_key(self.base_key)), + field_bytes(3, signal_public_key(self.identity_key)), + field_bytes(4, self.message), + field_varint(5, self.registration_id), + field_varint(6, self.signed_pre_key_id), + ) + ) + return b"".join(parts) + + def serialize(self) -> bytes: + return bytes((_version_byte(self.message_version),)) + self.protobuf() + + @classmethod + def parse(cls, raw: bytes) -> "PreKeyWhisperMessageV3": + data = bytes(raw) + if len(data) < 2: + raise SignalWireError("PreKeyWhisper message is too short") + version = _parse_version(data[0]) + fields = _field_map(data[1:]) + base = _required_bytes(fields, 2, "baseKey") + ident = _required_bytes(fields, 3, "identityKey") + if len(base) != 33 or base[0] != 5 or len(ident) != 33 or ident[0] != 5: + raise SignalWireError("PreKeyWhisper keys must use 0x05-prefixed Curve25519 encoding") + return cls( + registration_id=_required_int(fields, 5, "registrationId"), + pre_key_id=_optional_int(fields, 1), + signed_pre_key_id=_required_int(fields, 6, "signedPreKeyId"), + base_key=base, + identity_key=ident, + message=_required_bytes(fields, 4, "message"), + message_version=version, + ) diff --git a/src/postmaster/whatsapp_v990/store.py b/src/postmaster/whatsapp_v990/store.py new file mode 100644 index 00000000..6da18711 --- /dev/null +++ b/src/postmaster/whatsapp_v990/store.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +from contextlib import closing +import json +import os +from pathlib import Path +import sqlite3 +import stat +import threading +from typing import Any, Mapping + +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + +class AuthStoreError(RuntimeError): + pass + + +class EncryptedAuthStore: + """Persistent opaque WhatsApp auth/session state encrypted at rest. + + The generated master key is stored outside SQLite with mode 0600. No API returns the key or + decrypted private material wholesale; callers address explicit state namespaces and names. + """ + + def __init__(self, db_path: str, key_path: str | None = None): + self.db_path = str(db_path) + self.key_path = str(key_path or f"{db_path}.key") + Path(self.db_path).parent.mkdir(parents=True, exist_ok=True) + Path(self.key_path).parent.mkdir(parents=True, exist_ok=True) + self._lock = threading.RLock() + self._key = self._load_or_create_key() + self._init_db() + + def _load_or_create_key(self) -> bytes: + path = Path(self.key_path) + if path.exists(): + key = path.read_bytes() + if len(key) != 32: raise AuthStoreError("WhatsApp auth master key must be 32 bytes") + try: + mode = stat.S_IMODE(path.stat().st_mode) + if mode & 0o077: os.chmod(path, 0o600) + except OSError: + pass + return key + key = os.urandom(32) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(fd, "wb") as handle: handle.write(key) + except Exception: + try: path.unlink(missing_ok=True) + except OSError: pass + raise + return key + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path, timeout=30) + conn.row_factory = sqlite3.Row + return conn + + def _init_db(self) -> None: + with closing(self._connect()) as conn: + conn.execute("PRAGMA journal_mode=WAL") + conn.execute(""" + CREATE TABLE IF NOT EXISTS whatsapp_auth_state ( + namespace TEXT NOT NULL, + name TEXT NOT NULL, + nonce BLOB NOT NULL, + ciphertext BLOB NOT NULL, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(namespace, name) + ) + """) + conn.commit() + + @staticmethod + def _aad(namespace: str, name: str) -> bytes: + return f"postmaster-whatsapp-v990\0{namespace}\0{name}".encode("utf-8") + + def put_bytes(self, namespace: str, name: str, value: bytes) -> None: + ns, nm = str(namespace).strip(), str(name).strip() + if not ns or not nm: raise AuthStoreError("Auth state namespace and name are required") + nonce = os.urandom(12) + ciphertext = AESGCM(self._key).encrypt(nonce, bytes(value), self._aad(ns, nm)) + with self._lock, closing(self._connect()) as conn: + conn.execute( + "INSERT INTO whatsapp_auth_state(namespace,name,nonce,ciphertext,updated_at) VALUES(?,?,?,?,CURRENT_TIMESTAMP) " + "ON CONFLICT(namespace,name) DO UPDATE SET nonce=excluded.nonce,ciphertext=excluded.ciphertext,updated_at=CURRENT_TIMESTAMP", + (ns, nm, nonce, ciphertext), + ) + conn.commit() + + def get_bytes(self, namespace: str, name: str) -> bytes | None: + ns, nm = str(namespace).strip(), str(name).strip() + with self._lock, closing(self._connect()) as conn: + row = conn.execute("SELECT nonce,ciphertext FROM whatsapp_auth_state WHERE namespace=? AND name=?", (ns, nm)).fetchone() + if row is None: return None + try: + return AESGCM(self._key).decrypt(bytes(row["nonce"]), bytes(row["ciphertext"]), self._aad(ns, nm)) + except Exception as exc: + raise AuthStoreError("Unable to authenticate/decrypt WhatsApp auth state") from exc + + def put_json(self, namespace: str, name: str, value: Mapping[str, Any]) -> None: + self.put_bytes(namespace, name, json.dumps(dict(value), sort_keys=True, separators=(",", ":")).encode("utf-8")) + + def get_json(self, namespace: str, name: str) -> dict[str, Any] | None: + raw = self.get_bytes(namespace, name) + if raw is None: return None + value = json.loads(raw.decode("utf-8")) + if not isinstance(value, dict): raise AuthStoreError("Stored WhatsApp auth JSON must be an object") + return value + + def delete(self, namespace: str, name: str) -> bool: + with self._lock, closing(self._connect()) as conn: + cur = conn.execute("DELETE FROM whatsapp_auth_state WHERE namespace=? AND name=?", (str(namespace), str(name))) + conn.commit() + return cur.rowcount > 0 + + def status(self) -> dict[str, Any]: + with self._lock, closing(self._connect()) as conn: + count = int(conn.execute("SELECT COUNT(*) FROM whatsapp_auth_state").fetchone()[0]) + return {"ok": True, "encrypted_at_rest": True, "items": count, "private_material_exposed": False, "key_mode_target": "0600"} diff --git a/src/postmaster/whatsapp_v990/tokens.py b/src/postmaster/whatsapp_v990/tokens.py new file mode 100644 index 00000000..ccd39d41 --- /dev/null +++ b/src/postmaster/whatsapp_v990/tokens.py @@ -0,0 +1,1298 @@ +from __future__ import annotations + +"""Pinned WhatsApp WABinary token dictionary for Postmaster v9.9. + +This file contains protocol token *data* only, generated clean-room from the publicly +observable current WhatsApp Web token table snapshot used for interoperability validation. + +Source snapshot: WhiskeySockets/Baileys 0af2386292907f7d9742d8d41f830d8c48208fa1 +Source data file: src/WABinary/constants.ts +Single-byte tokens: 236; double-byte dictionaries: 4 x 256. + +Do not treat this pin as permanent: the live GitHub Action pair-device probe is expected +to fail closed when WhatsApp changes token assignments. +""" + +from .binary import TokenTable + +SINGLE_BYTE_TOKENS = [ + "", + "xmlstreamstart", + "xmlstreamend", + "s.whatsapp.net", + "type", + "participant", + "from", + "receipt", + "id", + "notification", + "disappearing_mode", + "status", + "jid", + "broadcast", + "user", + "devices", + "device_hash", + "to", + "offline", + "message", + "result", + "class", + "xmlns", + "duration", + "notify", + "iq", + "t", + "ack", + "g.us", + "enc", + "urn:xmpp:whatsapp:push", + "presence", + "config_value", + "picture", + "verified_name", + "config_code", + "key-index-list", + "contact", + "mediatype", + "routing_info", + "edge_routing", + "get", + "read", + "urn:xmpp:ping", + "fallback_hostname", + "0", + "chatstate", + "business_hours_config", + "unavailable", + "download_buckets", + "skmsg", + "verified_level", + "composing", + "handshake", + "device-list", + "media", + "text", + "fallback_ip4", + "media_conn", + "device", + "creation", + "location", + "config", + "item", + "fallback_ip6", + "count", + "w:profile:picture", + "image", + "business", + "2", + "hostname", + "call-creator", + "display_name", + "relaylatency", + "platform", + "abprops", + "success", + "msg", + "offline_preview", + "prop", + "key-index", + "v", + "day_of_week", + "pkmsg", + "version", + "1", + "ping", + "w:p", + "download", + "video", + "set", + "specific_hours", + "props", + "primary", + "unknown", + "hash", + "commerce_experience", + "last", + "subscribe", + "max_buckets", + "call", + "profile", + "member_since_text", + "close_time", + "call-id", + "sticker", + "mode", + "participants", + "value", + "query", + "profile_options", + "open_time", + "code", + "list", + "host", + "ts", + "contacts", + "upload", + "lid", + "preview", + "update", + "usync", + "w:stats", + "delivery", + "auth_ttl", + "context", + "fail", + "cart_enabled", + "appdata", + "category", + "atn", + "direct_connection", + "decrypt-fail", + "relay_id", + "mmg-fallback.whatsapp.net", + "target", + "available", + "name", + "last_id", + "mmg.whatsapp.net", + "categories", + "401", + "is_new", + "index", + "tctoken", + "ip4", + "token_id", + "latency", + "recipient", + "edit", + "ip6", + "add", + "thumbnail-document", + "26", + "paused", + "true", + "identity", + "stream:error", + "key", + "sidelist", + "background", + "audio", + "3", + "thumbnail-image", + "biz-cover-photo", + "cat", + "gcm", + "thumbnail-video", + "error", + "auth", + "deny", + "serial", + "in", + "registration", + "thumbnail-link", + "remove", + "00", + "gif", + "thumbnail-gif", + "tag", + "capability", + "multicast", + "item-not-found", + "description", + "business_hours", + "config_expo_key", + "md-app-state", + "expiration", + "fallback", + "ttl", + "300", + "md-msg-hist", + "device_orientation", + "out", + "w:m", + "open_24h", + "side_list", + "token", + "inactive", + "01", + "document", + "te2", + "played", + "encrypt", + "msgr", + "hide", + "direct_path", + "12", + "state", + "not-authorized", + "url", + "terminate", + "signature", + "status-revoke-delay", + "02", + "te", + "linked_accounts", + "trusted_contact", + "timezone", + "ptt", + "kyc-id", + "privacy_token", + "readreceipts", + "appointment_only", + "address", + "expected_ts", + "privacy", + "7", + "android", + "interactive", + "device-identity", + "enabled", + "attribute_padding", + "1080", + "03", + "screen_height" +] + +DOUBLE_BYTE_TOKENS = [ + [ + "read-self", + "active", + "fbns", + "protocol", + "reaction", + "screen_width", + "heartbeat", + "deviceid", + "2:47DEQpj8", + "uploadfieldstat", + "voip_settings", + "retry", + "priority", + "longitude", + "conflict", + "false", + "ig_professional", + "replaced", + "preaccept", + "cover_photo", + "uncompressed", + "encopt", + "ppic", + "04", + "passive", + "status-revoke-drop", + "keygen", + "540", + "offer", + "rate", + "opus", + "latitude", + "w:gp2", + "ver", + "4", + "business_profile", + "medium", + "sender", + "prev_v_id", + "email", + "website", + "invited", + "sign_credential", + "05", + "transport", + "skey", + "reason", + "peer_abtest_bucket", + "America/Sao_Paulo", + "appid", + "refresh", + "100", + "06", + "404", + "101", + "104", + "107", + "102", + "109", + "103", + "member_add_mode", + "105", + "transaction-id", + "110", + "106", + "outgoing", + "108", + "111", + "tokens", + "followers", + "ig_handle", + "self_pid", + "tue", + "dec", + "thu", + "joinable", + "peer_pid", + "mon", + "features", + "wed", + "peer_device_presence", + "pn", + "delete", + "07", + "fri", + "audio_duration", + "admin", + "connected", + "delta", + "rcat", + "disable", + "collection", + "08", + "480", + "sat", + "phash", + "all", + "invite", + "accept", + "critical_unblock_low", + "group_update", + "signed_credential", + "blinded_credential", + "eph_setting", + "net", + "09", + "background_location", + "refresh_id", + "Asia/Kolkata", + "privacy_mode_ts", + "account_sync", + "voip_payload_type", + "service_areas", + "acs_public_key", + "v_id", + "0a", + "fallback_class", + "relay", + "actual_actors", + "metadata", + "w:biz", + "5", + "connected-limit", + "notice", + "0b", + "host_storage", + "fb_page", + "subject", + "privatestats", + "invis", + "groupadd", + "010", + "note.m4r", + "uuid", + "0c", + "8000", + "sun", + "372", + "1020", + "stage", + "1200", + "720", + "canonical", + "fb", + "011", + "video_duration", + "0d", + "1140", + "superadmin", + "012", + "Opening.m4r", + "keystore_attestation", + "dleq_proof", + "013", + "timestamp", + "ab_key", + "w:sync:app:state", + "0e", + "vertical", + "600", + "p_v_id", + "6", + "likes", + "014", + "500", + "1260", + "creator", + "0f", + "rte", + "destination", + "group", + "group_info", + "syncd_anti_tampering_fatal_exception_enabled", + "015", + "dl_bw", + "Asia/Jakarta", + "vp8/h.264", + "online", + "1320", + "fb:multiway", + "10", + "timeout", + "016", + "nse_retry", + "urn:xmpp:whatsapp:dirty", + "017", + "a_v_id", + "web_shops_chat_header_button_enabled", + "nse_call", + "inactive-upgrade", + "none", + "web", + "groups", + "2250", + "mms_hot_content_timespan_in_seconds", + "contact_blacklist", + "nse_read", + "suspended_group_deletion_notification", + "binary_version", + "018", + "https://www.whatsapp.com/otp/copy/", + "reg_push", + "shops_hide_catalog_attachment_entrypoint", + "server_sync", + ".", + "ephemeral_messages_allowed_values", + "019", + "mms_vcache_aggregation_enabled", + "iphone", + "America/Argentina/Buenos_Aires", + "01a", + "mms_vcard_autodownload_size_kb", + "nse_ver", + "shops_header_dropdown_menu_item", + "dhash", + "catalog_status", + "communities_mvp_new_iqs_serverprop", + "blocklist", + "default", + "11", + "ephemeral_messages_enabled", + "01b", + "original_dimensions", + "8", + "mms4_media_retry_notification_encryption_enabled", + "mms4_server_error_receipt_encryption_enabled", + "original_image_url", + "sync", + "multiway", + "420", + "companion_enc_static", + "shops_profile_drawer_entrypoint", + "01c", + "vcard_as_document_size_kb", + "status_video_max_duration", + "request_image_url", + "01d", + "regular_high", + "s_t", + "abt", + "share_ext_min_preliminary_image_quality", + "01e", + "32", + "syncd_key_rotation_enabled", + "data_namespace", + "md_downgrade_read_receipts2", + "patch", + "polltype", + "ephemeral_messages_setting", + "userrate", + "15", + "partial_pjpeg_bw_threshold", + "played-self", + "catalog_exists", + "01f", + "mute_v2" + ], + [ + "reject", + "dirty", + "announcement", + "020", + "13", + "9", + "status_video_max_bitrate", + "fb:thrift_iq", + "offline_batch", + "022", + "full", + "ctwa_first_business_reply_logging", + "h.264", + "smax_id", + "group_description_length", + "https://www.whatsapp.com/otp/code", + "status_image_max_edge", + "smb_upsell_business_profile_enabled", + "021", + "web_upgrade_to_md_modal", + "14", + "023", + "s_o", + "smaller_video_thumbs_status_enabled", + "media_max_autodownload", + "960", + "blocking_status", + "peer_msg", + "joinable_group_call_client_version", + "group_call_video_maximization_enabled", + "return_snapshot", + "high", + "America/Mexico_City", + "entry_point_block_logging_enabled", + "pop", + "024", + "1050", + "16", + "1380", + "one_tap_calling_in_group_chat_size", + "regular_low", + "inline_joinable_education_enabled", + "hq_image_max_edge", + "locked", + "America/Bogota", + "smb_biztools_deeplink_enabled", + "status_image_quality", + "1088", + "025", + "payments_upi_intent_transaction_limit", + "voip", + "w:g2", + "027", + "md_pin_chat_enabled", + "026", + "multi_scan_pjpeg_download_enabled", + "shops_product_grid", + "transaction_id", + "ctwa_context_enabled", + "20", + "fna", + "hq_image_quality", + "alt_jpeg_doc_detection_quality", + "group_call_max_participants", + "pkey", + "America/Belem", + "image_max_kbytes", + "web_cart_v1_1_order_message_changes_enabled", + "ctwa_context_enterprise_enabled", + "urn:xmpp:whatsapp:account", + "840", + "Asia/Kuala_Lumpur", + "max_participants", + "video_remux_after_repair_enabled", + "stella_addressbook_restriction_type", + "660", + "900", + "780", + "context_menu_ios13_enabled", + "mute-state", + "ref", + "payments_request_messages", + "029", + "frskmsg", + "vcard_max_size_kb", + "sample_buffer_gif_player_enabled", + "match_last_seen", + "510", + "4983", + "video_max_bitrate", + "028", + "w:comms:chat", + "17", + "frequently_forwarded_max", + "groups_privacy_blacklist", + "Asia/Karachi", + "02a", + "web_download_document_thumb_mms_enabled", + "02b", + "hist_sync", + "biz_block_reasons_version", + "1024", + "18", + "web_is_direct_connection_for_plm_transparent", + "view_once_write", + "file_max_size", + "paid_convo_id", + "online_privacy_setting", + "video_max_edge", + "view_once_read", + "enhanced_storage_management", + "multi_scan_pjpeg_encoding_enabled", + "ctwa_context_forward_enabled", + "video_transcode_downgrade_enable", + "template_doc_mime_types", + "hq_image_bw_threshold", + "30", + "body", + "u_aud_limit_sil_restarts_ctrl", + "other", + "participating", + "w:biz:directory", + "1110", + "vp8", + "4018", + "meta", + "doc_detection_image_max_edge", + "image_quality", + "1170", + "02c", + "smb_upsell_chat_banner_enabled", + "key_expiry_time_second", + "pid", + "stella_interop_enabled", + "19", + "linked_device_max_count", + "md_device_sync_enabled", + "02d", + "02e", + "360", + "enhanced_block_enabled", + "ephemeral_icon_in_forwarding", + "paid_convo_status", + "gif_provider", + "project_name", + "server-error", + "canonical_url_validation_enabled", + "wallpapers_v2", + "syncd_clear_chat_delete_chat_enabled", + "medianotify", + "02f", + "shops_required_tos_version", + "vote", + "reset_skey_on_id_change", + "030", + "image_max_edge", + "multicast_limit_global", + "ul_bw", + "21", + "25", + "5000", + "poll", + "570", + "22", + "031", + "1280", + "WhatsApp", + "032", + "bloks_shops_enabled", + "50", + "upload_host_switching_enabled", + "web_ctwa_context_compose_enabled", + "ptt_forwarded_features_enabled", + "unblocked", + "partial_pjpeg_enabled", + "fbid:devices", + "height", + "ephemeral_group_query_ts", + "group_join_permissions", + "order", + "033", + "alt_jpeg_status_quality", + "migrate", + "popular-bank", + "win_uwp_deprecation_killswitch_enabled", + "web_download_status_thumb_mms_enabled", + "blocking", + "url_text", + "035", + "web_forwarding_limit_to_groups", + "1600", + "val", + "1000", + "syncd_msg_date_enabled", + "bank-ref-id", + "max_subject", + "payments_web_enabled", + "web_upload_document_thumb_mms_enabled", + "size", + "request", + "ephemeral", + "24", + "receipt_agg", + "ptt_remember_play_position", + "sampling_weight", + "enc_rekey", + "mute_always", + "037", + "034", + "23", + "036", + "action", + "click_to_chat_qr_enabled", + "width", + "disabled", + "038", + "md_blocklist_v2", + "played_self_enabled", + "web_buttons_message_enabled", + "flow_id", + "clear", + "450", + "fbid:thread", + "bloks_session_state", + "America/Lima", + "attachment_picker_refresh", + "download_host_switching_enabled", + "1792", + "u_aud_limit_sil_restarts_test2", + "custom_urls", + "device_fanout", + "optimistic_upload", + "2000", + "key_cipher_suite", + "web_smb_upsell_in_biz_profile_enabled", + "e", + "039", + "siri_post_status_shortcut", + "pair-device", + "lg", + "lc", + "stream_attribution_url", + "model", + "mspjpeg_phash_gen", + "catalog_send_all", + "new_multi_vcards_ui", + "share_biz_vcard_enabled", + "-", + "clean", + "200", + "md_blocklist_v2_server", + "03b", + "03a", + "web_md_migration_experience", + "ptt_conversation_waveform", + "u_aud_limit_sil_restarts_test1" + ], + [ + "64", + "ptt_playback_speed_enabled", + "web_product_list_message_enabled", + "paid_convo_ts", + "27", + "manufacturer", + "psp-routing", + "grp_uii_cleanup", + "ptt_draft_enabled", + "03c", + "business_initiated", + "web_catalog_products_onoff", + "web_upload_link_thumb_mms_enabled", + "03e", + "mediaretry", + "35", + "hfm_string_changes", + "28", + "America/Fortaleza", + "max_keys", + "md_mhfs_days", + "streaming_upload_chunk_size", + "5541", + "040", + "03d", + "2675", + "03f", + "...", + "512", + "mute", + "48", + "041", + "alt_jpeg_quality", + "60", + "042", + "md_smb_quick_reply", + "5183", + "c", + "1343", + "40", + "1230", + "043", + "044", + "mms_cat_v1_forward_hot_override_enabled", + "user_notice", + "ptt_waveform_send", + "047", + "Asia/Calcutta", + "250", + "md_privacy_v2", + "31", + "29", + "128", + "md_messaging_enabled", + "046", + "crypto", + "690", + "045", + "enc_iv", + "75", + "failure", + "ptt_oot_playback", + "AIzaSyDR5yfaG7OG8sMTUj8kfQEb8T9pN8BM6Lk", + "w", + "048", + "2201", + "web_large_files_ui", + "Asia/Makassar", + "812", + "status_collapse_muted", + "1334", + "257", + "2HP4dm", + "049", + "patches", + "1290", + "43cY6T", + "America/Caracas", + "web_sticker_maker", + "campaign", + "ptt_pausable_enabled", + "33", + "42", + "attestation", + "biz", + "04b", + "query_linked", + "s", + "125", + "04a", + "810", + "availability", + "1411", + "responsiveness_v2_m1", + "catalog_not_created", + "34", + "America/Santiago", + "1465", + "enc_p", + "04d", + "status_info", + "04f", + "key_version", + "..", + "04c", + "04e", + "md_group_notification", + "1598", + "1215", + "web_cart_enabled", + "37", + "630", + "1920", + "2394", + "-1", + "vcard", + "38", + "elapsed", + "36", + "828", + "peer", + "pricing_category", + "1245", + "invalid", + "stella_ios_enabled", + "2687", + "45", + "1528", + "39", + "u_is_redial_audio_1104_ctrl", + "1025", + "1455", + "58", + "2524", + "2603", + "054", + "bsp_system_message_enabled", + "web_pip_redesign", + "051", + "verify_apps", + "1974", + "1272", + "1322", + "1755", + "052", + "70", + "050", + "1063", + "1135", + "1361", + "80", + "1096", + "1828", + "1851", + "1251", + "1921", + "key_config_id", + "1254", + "1566", + "1252", + "2525", + "critical_block", + "1669", + "max_available", + "w:auth:backup:token", + "product", + "2530", + "870", + "1022", + "participant_uuid", + "web_cart_on_off", + "1255", + "1432", + "1867", + "41", + "1415", + "1440", + "240", + "1204", + "1608", + "1690", + "1846", + "1483", + "1687", + "1749", + "69", + "url_number", + "053", + "1325", + "1040", + "365", + "59", + "Asia/Riyadh", + "1177", + "test_recommended", + "057", + "1612", + "43", + "1061", + "1518", + "1635", + "055", + "1034", + "1375", + "750", + "1430", + "event_code", + "1682", + "503", + "55", + "865", + "78", + "1309", + "1365", + "44", + "America/Guayaquil", + "535", + "LIMITED", + "1377", + "1613", + "1420", + "1599", + "1822", + "05a", + "1681", + "password", + "1111", + "1214", + "1376", + "1478", + "47", + "1082", + "4282", + "Europe/Istanbul", + "1307", + "46", + "058", + "1124", + "256", + "rate-overlimit", + "retail", + "u_a_socket_err_fix_succ_test", + "1292", + "1370", + "1388", + "520", + "861", + "psa", + "regular", + "1181", + "1766", + "05b", + "1183", + "1213", + "1304", + "1537" + ], + [ + "1724", + "profile_picture", + "1071", + "1314", + "1605", + "407", + "990", + "1710", + "746", + "pricing_model", + "056", + "059", + "061", + "1119", + "6027", + "65", + "877", + "1607", + "05d", + "917", + "seen", + "1516", + "49", + "470", + "973", + "1037", + "1350", + "1394", + "1480", + "1796", + "keys", + "794", + "1536", + "1594", + "2378", + "1333", + "1524", + "1825", + "116", + "309", + "52", + "808", + "827", + "909", + "495", + "1660", + "361", + "957", + "google", + "1357", + "1565", + "1967", + "996", + "1775", + "586", + "736", + "1052", + "1670", + "bank", + "177", + "1416", + "2194", + "2222", + "1454", + "1839", + "1275", + "53", + "997", + "1629", + "6028", + "smba", + "1378", + "1410", + "05c", + "1849", + "727", + "create", + "1559", + "536", + "1106", + "1310", + "1944", + "670", + "1297", + "1316", + "1762", + "en", + "1148", + "1295", + "1551", + "1853", + "1890", + "1208", + "1784", + "7200", + "05f", + "178", + "1283", + "1332", + "381", + "643", + "1056", + "1238", + "2024", + "2387", + "179", + "981", + "1547", + "1705", + "05e", + "290", + "903", + "1069", + "1285", + "2436", + "062", + "251", + "560", + "582", + "719", + "56", + "1700", + "2321", + "325", + "448", + "613", + "777", + "791", + "51", + "488", + "902", + "Asia/Almaty", + "is_hidden", + "1398", + "1527", + "1893", + "1999", + "2367", + "2642", + "237", + "busy", + "065", + "067", + "233", + "590", + "993", + "1511", + "54", + "723", + "860", + "363", + "487", + "522", + "605", + "995", + "1321", + "1691", + "1865", + "2447", + "2462", + "NON_TRANSACTIONAL", + "433", + "871", + "432", + "1004", + "1207", + "2032", + "2050", + "2379", + "2446", + "279", + "636", + "703", + "904", + "248", + "370", + "691", + "700", + "1068", + "1655", + "2334", + "060", + "063", + "364", + "533", + "534", + "567", + "1191", + "1210", + "1473", + "1827", + "069", + "701", + "2531", + "514", + "prev_dhash", + "064", + "496", + "790", + "1046", + "1139", + "1505", + "1521", + "1108", + "207", + "544", + "637", + "final", + "1173", + "1293", + "1694", + "1939", + "1951", + "1993", + "2353", + "2515", + "504", + "601", + "857", + "modify", + "spam_request", + "p_121_aa_1101_test4", + "866", + "1427", + "1502", + "1638", + "1744", + "2153", + "068", + "382", + "725", + "1704", + "1864", + "1990", + "2003", + "Asia/Dubai", + "508", + "531", + "1387", + "1474", + "1632", + "2307", + "2386", + "819", + "2014", + "066", + "387", + "1468", + "1706", + "2186", + "2261", + "471", + "728", + "1147", + "1372", + "1961" + ] +] + +CURRENT_TOKEN_TABLE = TokenTable( + SINGLE_BYTE_TOKENS, + DOUBLE_BYTE_TOKENS, + version=20260921, +) + +__all__ = ["SINGLE_BYTE_TOKENS", "DOUBLE_BYTE_TOKENS", "CURRENT_TOKEN_TABLE"] diff --git a/src/postmaster/whatsapp_v990/transport.py b/src/postmaster/whatsapp_v990/transport.py new file mode 100644 index 00000000..37a34f69 --- /dev/null +++ b/src/postmaster/whatsapp_v990/transport.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +import random +import time +from typing import Awaitable, Callable, Protocol + + +class TransportError(RuntimeError): + pass + + +class WebSocketLike(Protocol): + async def send(self, data: bytes) -> object: ... + async def recv(self) -> bytes | str: ... + async def close(self) -> object: ... + + +Connector = Callable[[], Awaitable[WebSocketLike]] + + +@dataclass(slots=True) +class ReconnectPolicy: + base_seconds: float = 1.0 + max_seconds: float = 30.0 + factor: float = 1.7 + jitter: float = 0.2 + + def delay(self, attempt: int, *, rand: Callable[[], float] = random.random) -> float: + raw = min(self.max_seconds, self.base_seconds * (self.factor ** max(0, int(attempt)))) + spread = raw * max(0.0, self.jitter) + return max(0.0, raw - spread + (2 * spread * rand())) + + +class ReconnectingTransport: + """Async WebSocket lifecycle with injected connector and explicit stop semantics. + + Network/library choice stays outside this module, keeping the clean-room protocol core pure + Python and avoiding a hard requirements.txt dependency. + """ + + def __init__(self, connector: Connector, *, policy: ReconnectPolicy | None = None): + self.connector = connector + self.policy = policy or ReconnectPolicy() + self.socket: WebSocketLike | None = None + self.connected = False + self.connect_attempts = 0 + self.reconnects = 0 + self.last_error: str | None = None + self.last_connected_at: float | None = None + self._stop = False + self._lock = asyncio.Lock() + + async def connect(self) -> None: + async with self._lock: + if self.connected and self.socket is not None: return + self.connect_attempts += 1 + try: + self.socket = await self.connector() + self.connected = True + self.last_error = None + self.last_connected_at = time.time() + except Exception as exc: + self.connected = False; self.socket = None; self.last_error = f"{type(exc).__name__}: {exc}" + raise TransportError(self.last_error) from exc + + async def ensure_connected(self, *, max_attempts: int | None = None, sleep: Callable[[float], Awaitable[object]] = asyncio.sleep) -> None: + attempt = 0 + while not self._stop: + try: + await self.connect(); return + except TransportError: + if max_attempts is not None and attempt + 1 >= max_attempts: raise + await sleep(self.policy.delay(attempt)); attempt += 1; self.reconnects += 1 + raise TransportError("Transport stopped") + + async def send(self, payload: bytes) -> None: + if not self.connected or self.socket is None: raise TransportError("WhatsApp transport is not connected") + try: + await self.socket.send(bytes(payload)) + except Exception as exc: + self.connected = False; self.last_error = f"{type(exc).__name__}: {exc}" + raise TransportError(self.last_error) from exc + + async def recv(self) -> bytes: + if not self.connected or self.socket is None: raise TransportError("WhatsApp transport is not connected") + try: + value = await self.socket.recv() + except Exception as exc: + self.connected = False; self.last_error = f"{type(exc).__name__}: {exc}" + raise TransportError(self.last_error) from exc + return value.encode("utf-8") if isinstance(value, str) else bytes(value) + + async def close(self) -> None: + self._stop = True + sock, self.socket = self.socket, None + self.connected = False + if sock is not None: + await sock.close() + + def status(self) -> dict[str, object]: + return { + "connected": self.connected, + "connect_attempts": self.connect_attempts, + "reconnects": self.reconnects, + "last_error": self.last_error, + "last_connected_at": self.last_connected_at, + "stopped": self._stop, + } diff --git a/src/postmaster/whatsapp_v990/usync.py b/src/postmaster/whatsapp_v990/usync.py new file mode 100644 index 00000000..db3ccb77 --- /dev/null +++ b/src/postmaster/whatsapp_v990/usync.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +"""Minimal WhatsApp USync device/LID discovery for message fanout.""" + +from dataclasses import dataclass +import secrets +from typing import Iterable + +from .binary import BinaryNode +from .jid import JID, parse_jid + + +class USyncError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class DeviceTarget: + jid: str + device: int + key_index: int | None + hosted: bool + source_jid: str + lid: str | None = None + + +def build_device_query( + jids: Iterable[str], + *, + stanza_id: str | None = None, + sid: str | None = None, + context: str = "message", +) -> BinaryNode: + users: list[BinaryNode] = [] + seen: set[str] = set() + for value in jids: + parsed = parse_jid(str(value)).normalized_user() + bare = parsed.bare + if bare in seen: + continue + seen.add(bare) + users.append(BinaryNode("user", {"jid": bare})) + if not users: + raise USyncError("USync device query requires at least one JID") + attrs = {"to": "s.whatsapp.net", "type": "get", "xmlns": "usync"} + if stanza_id: + attrs["id"] = stanza_id + return BinaryNode( + "iq", + attrs, + [ + BinaryNode( + "usync", + { + "context": str(context), + "mode": "query", + "sid": sid or ("pm-" + secrets.token_hex(8)), + "last": "true", + "index": "0", + }, + [ + BinaryNode("query", {}, [BinaryNode("devices", {"version": "2"}), BinaryNode("lid")]), + BinaryNode("list", {}, users), + ], + ) + ], + ) + + +def _server_for(base: JID, hosted: bool) -> str: + if base.is_lid: + return "hosted.lid" if hosted else "lid" + return "hosted" if hosted else "s.whatsapp.net" + + +def parse_device_result( + response: BinaryNode, + *, + own_jid: str | None = None, + ignore_zero_devices: bool = False, + prefer_lid: bool = True, +) -> list[DeviceTarget]: + if response.attrs.get("type") != "result": + raise USyncError("USync response is not a result") + usync = response.child("usync") + listing = usync.child("list") if usync is not None else None + if listing is None: + raise USyncError("USync result has no list") + own = parse_jid(own_jid) if own_jid else None + targets: list[DeviceTarget] = [] + seen: set[str] = set() + for user in listing.children("user"): + source = str(user.attrs.get("jid") or "").strip() + if not source: + continue + base = parse_jid(source).normalized_user() + lid_node = user.child("lid") + lid_value = str(lid_node.attrs.get("val") or "").strip() if lid_node else "" + lid_base: JID | None = None + if lid_value: + try: + lid_base = parse_jid(lid_value).normalized_user() + except ValueError: + lid_base = None + devices = user.child("devices") + device_list = devices.child("device-list") if devices is not None else None + if device_list is None: + continue + for node in device_list.children("device"): + try: + device = int(node.attrs.get("id", "")) + except ValueError: + continue + if not 0 <= device <= 255: + continue + if ignore_zero_devices and device == 0: + continue + key_index: int | None + try: + key_index = int(node.attrs["key-index"]) if node.attrs.get("key-index") not in (None, "") else None + except ValueError: + key_index = None + if device != 0 and key_index is None: + continue + hosted = str(node.attrs.get("is_hosted") or "").lower() == "true" + chosen = lid_base if prefer_lid and lid_base is not None else base + server = _server_for(chosen, hosted) + target = JID(chosen.user, server, device=device, domain_type=chosen.domain_type) + if own is not None and target.user == own.user and int(target.device or 0) == int(own.device or 0): + continue + text = str(target) + if text in seen: + continue + seen.add(text) + targets.append( + DeviceTarget( + jid=text, + device=device, + key_index=key_index, + hosted=hosted, + source_jid=base.bare, + lid=lid_base.bare if lid_base is not None else None, + ) + ) + return targets + + +__all__ = ["USyncError", "DeviceTarget", "build_device_query", "parse_device_result"] diff --git a/src/postmaster/whatsapp_v990/websocket_driver.py b/src/postmaster/whatsapp_v990/websocket_driver.py new file mode 100644 index 00000000..e3a0b607 --- /dev/null +++ b/src/postmaster/whatsapp_v990/websocket_driver.py @@ -0,0 +1,288 @@ +from __future__ import annotations + +import asyncio +import base64 +from dataclasses import dataclass +import hashlib +import os +import ssl +from typing import Any, Awaitable, Callable +from urllib.parse import urlparse + + +DEFAULT_WA_WEBSOCKET_URL = "wss://web.whatsapp.com/ws/chat" +DEFAULT_WA_ORIGIN = "https://web.whatsapp.com" +_WS_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" +_MAX_HTTP_HEADERS = 64 * 1024 + + +class WebSocketDriverError(RuntimeError): + pass + + +@dataclass(frozen=True, slots=True) +class WebSocketDriverConfig: + url: str = DEFAULT_WA_WEBSOCKET_URL + origin: str = DEFAULT_WA_ORIGIN + open_timeout: float = 20.0 + close_timeout: float = 10.0 + max_size: int | None = None + prefer_native: bool = False + + +class NativeWebSocket: + """Small RFC6455 client sufficient for the WhatsApp binary transport. + + Client frames are always masked as required by RFC6455. TLS certificate verification is + performed by asyncio.open_connection's default SSL context. The reader handles ping/pong, + close and fragmented text/binary messages; extensions are deliberately not negotiated. + """ + + def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, *, max_size: int | None = None): + self.reader = reader + self.writer = writer + self.max_size = max_size + self.closed = False + self._send_lock = asyncio.Lock() + self._recv_lock = asyncio.Lock() + + async def _send_frame(self, opcode: int, payload: bytes = b"") -> None: + if self.closed and opcode != 0x8: + raise WebSocketDriverError("WebSocket is closed") + raw = bytes(payload) + if opcode >= 0x8 and len(raw) > 125: + raise WebSocketDriverError("WebSocket control frame exceeds 125 bytes") + mask = os.urandom(4) + first = 0x80 | (opcode & 0x0F) + length = len(raw) + if length < 126: + header = bytes((first, 0x80 | length)) + elif length <= 0xFFFF: + header = bytes((first, 0x80 | 126)) + length.to_bytes(2, "big") + else: + if length >= (1 << 63): + raise WebSocketDriverError("WebSocket payload is too large") + header = bytes((first, 0x80 | 127)) + length.to_bytes(8, "big") + masked = bytes(value ^ mask[i & 3] for i, value in enumerate(raw)) + async with self._send_lock: + self.writer.write(header + mask + masked) + await self.writer.drain() + + async def send(self, data: bytes) -> None: + await self._send_frame(0x2, bytes(data)) + + async def _read_frame(self) -> tuple[bool, int, bytes]: + try: + head = await self.reader.readexactly(2) + except (asyncio.IncompleteReadError, ConnectionError) as exc: + self.closed = True + raise WebSocketDriverError("WebSocket connection ended while reading frame header") from exc + fin = bool(head[0] & 0x80) + rsv = head[0] & 0x70 + opcode = head[0] & 0x0F + masked = bool(head[1] & 0x80) + length = head[1] & 0x7F + if rsv: + raise WebSocketDriverError("Unexpected WebSocket RSV bits without negotiated extension") + if masked: + raise WebSocketDriverError("Server WebSocket frames must not be masked") + if length == 126: + length = int.from_bytes(await self.reader.readexactly(2), "big") + elif length == 127: + length = int.from_bytes(await self.reader.readexactly(8), "big") + if length >= (1 << 63): + raise WebSocketDriverError("Invalid WebSocket 64-bit payload length") + if opcode >= 0x8 and (not fin or length > 125): + raise WebSocketDriverError("Invalid fragmented/oversized WebSocket control frame") + if self.max_size is not None and length > self.max_size: + raise WebSocketDriverError("WebSocket frame exceeds configured max_size") + payload = await self.reader.readexactly(length) + return fin, opcode, payload + + async def recv(self) -> bytes | str: + async with self._recv_lock: + message_opcode: int | None = None + chunks: list[bytes] = [] + total = 0 + while True: + fin, opcode, payload = await self._read_frame() + if opcode == 0x8: + if not self.closed: + try: + await self._send_frame(0x8, payload[:125]) + except Exception: + pass + self.closed = True + raise WebSocketDriverError("WebSocket peer closed the connection") + if opcode == 0x9: + await self._send_frame(0xA, payload) + continue + if opcode == 0xA: + continue + if opcode in (0x1, 0x2): + if message_opcode is not None: + raise WebSocketDriverError("New WebSocket data frame before fragmented message completed") + message_opcode = opcode + chunks = [payload] + total = len(payload) + elif opcode == 0x0: + if message_opcode is None: + raise WebSocketDriverError("Unexpected WebSocket continuation frame") + chunks.append(payload) + total += len(payload) + else: + raise WebSocketDriverError(f"Unsupported WebSocket opcode {opcode}") + if self.max_size is not None and total > self.max_size: + raise WebSocketDriverError("WebSocket message exceeds configured max_size") + if fin: + data = b"".join(chunks) + if message_opcode == 0x1: + try: + return data.decode("utf-8") + except UnicodeDecodeError as exc: + raise WebSocketDriverError("Invalid UTF-8 WebSocket text frame") from exc + return data + + async def close(self) -> None: + if self.closed: + self.writer.close() + try: + await self.writer.wait_closed() + except Exception: + pass + return + try: + await self._send_frame(0x8, (1000).to_bytes(2, "big")) + except Exception: + pass + self.closed = True + self.writer.close() + try: + await self.writer.wait_closed() + except Exception: + pass + + +async def _open_native(cfg: WebSocketDriverConfig) -> NativeWebSocket: + parsed = urlparse(cfg.url) + if parsed.scheme != "wss" or not parsed.hostname: + raise WebSocketDriverError("WhatsApp WebSocket URL must use wss://") + host = parsed.hostname + port = parsed.port or 443 + path = parsed.path or "/" + if parsed.query: + path += "?" + parsed.query + context = ssl.create_default_context() + try: + reader, writer = await asyncio.wait_for( + asyncio.open_connection(host, port, ssl=context, server_hostname=host), + timeout=cfg.open_timeout, + ) + except Exception as exc: + raise WebSocketDriverError(f"WhatsApp TLS connection failed: {type(exc).__name__}: {exc}") from exc + + key = base64.b64encode(os.urandom(16)).decode("ascii") + host_header = host if port == 443 else f"{host}:{port}" + request = ( + f"GET {path} HTTP/1.1\r\n" + f"Host: {host_header}\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + f"Sec-WebSocket-Key: {key}\r\n" + "Sec-WebSocket-Version: 13\r\n" + f"Origin: {cfg.origin}\r\n" + "User-Agent: Postmaster-MCP-v9.9/clean-room-python\r\n" + "\r\n" + ).encode("ascii") + writer.write(request) + await writer.drain() + try: + header_block = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), timeout=cfg.open_timeout) + except Exception as exc: + writer.close() + raise WebSocketDriverError("WhatsApp WebSocket HTTP upgrade did not complete") from exc + if len(header_block) > _MAX_HTTP_HEADERS: + writer.close() + raise WebSocketDriverError("WhatsApp WebSocket HTTP headers exceed safety bound") + text = header_block.decode("iso-8859-1", errors="replace") + lines = text.split("\r\n") + status = lines[0] if lines else "" + headers: dict[str, str] = {} + for line in lines[1:]: + if ":" in line: + name, value = line.split(":", 1) + headers[name.strip().lower()] = value.strip() + expected_accept = base64.b64encode(hashlib.sha1((key + _WS_GUID).encode("ascii")).digest()).decode("ascii") + if not status.startswith("HTTP/1.1 101"): + writer.close() + raise WebSocketDriverError(f"WhatsApp WebSocket upgrade rejected: {status or 'no status'}") + if headers.get("upgrade", "").lower() != "websocket" or "upgrade" not in headers.get("connection", "").lower(): + writer.close() + raise WebSocketDriverError("WhatsApp WebSocket upgrade headers are invalid") + if headers.get("sec-websocket-accept", "") != expected_accept: + writer.close() + raise WebSocketDriverError("WhatsApp WebSocket Sec-WebSocket-Accept mismatch") + return NativeWebSocket(reader, writer, max_size=cfg.max_size) + + +async def open_whatsapp_websocket( + config: WebSocketDriverConfig | None = None, + *, + connect_impl: Callable[..., Awaitable[Any]] | None = None, +) -> Any: + """Open the raw WhatsApp WebSocket using injected, native, or optional library transport.""" + cfg = config or WebSocketDriverConfig() + if not str(cfg.url).startswith("wss://"): + raise WebSocketDriverError("WhatsApp WebSocket URL must use wss://") + if connect_impl is not None: + try: + return await connect_impl( + cfg.url, + origin=cfg.origin, + open_timeout=cfg.open_timeout, + close_timeout=cfg.close_timeout, + max_size=cfg.max_size, + ) + except Exception as exc: + raise WebSocketDriverError(f"WhatsApp WebSocket connection failed: {type(exc).__name__}: {exc}") from exc + + if cfg.prefer_native: + return await _open_native(cfg) + + try: + from websockets.asyncio.client import connect + except Exception: + return await _open_native(cfg) + try: + return await connect( + cfg.url, + origin=cfg.origin, + open_timeout=cfg.open_timeout, + close_timeout=cfg.close_timeout, + max_size=cfg.max_size, + ) + except Exception as exc: + raise WebSocketDriverError(f"WhatsApp WebSocket connection failed: {type(exc).__name__}: {exc}") from exc + + +def driver_status() -> dict[str, object]: + try: + import websockets # noqa: F401 + library_available = True + except Exception: + library_available = False + return { + "endpoint": DEFAULT_WA_WEBSOCKET_URL, + "origin": DEFAULT_WA_ORIGIN, + "driver_available": True, + "native_driver": True, + "optional_websockets_library": library_available, + "noise_required": True, + "protocol_authenticated": False, + } + + +__all__ = [ + "DEFAULT_WA_WEBSOCKET_URL", "DEFAULT_WA_ORIGIN", "WebSocketDriverError", "WebSocketDriverConfig", + "NativeWebSocket", "open_whatsapp_websocket", "driver_status", +] diff --git a/tests/test_calendar_v990.py b/tests/test_calendar_v990.py new file mode 100644 index 00000000..5a40c53f --- /dev/null +++ b/tests/test_calendar_v990.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from email.message import EmailMessage +import unittest + +from postmaster.calendar_v990 import build_calendar_request, detect_calendar_parts, parse_ics + + +class CalendarV990Tests(unittest.TestCase): + def test_roundtrip_request_and_video_link(self): + text = build_calendar_request( + uid="job-123@postmaster", + summary="Project sync", + start_utc=datetime(2026, 9, 21, 9, 0, tzinfo=UTC), + end_utc=datetime(2026, 9, 21, 9, 30, tzinfo=UTC), + organizer_email="owner@example.com", + attendees=["guest@example.com"], + description="Agenda", + video_url="https://meet.google.com/abc-defg-hij", + ) + parsed = parse_ics(text) + self.assertTrue(parsed["is_invitation"]) + event = parsed["events"][0] + self.assertEqual(event["uid"], "job-123@postmaster") + self.assertEqual(event["summary"], "Project sync") + self.assertEqual(event["attendees"][0]["address"], "guest@example.com") + self.assertEqual(event["video_urls"], ["https://meet.google.com/abc-defg-hij"]) + + def test_mime_inspection_is_local_and_detects_inline_calendar(self): + msg = EmailMessage() + msg["Subject"] = "Invite" + msg["From"] = "owner@example.com" + msg["To"] = "guest@example.com" + msg.set_content("Calendar invitation") + ics = build_calendar_request( + uid="u1@example.com", + summary="Demo", + start_utc=datetime(2026, 9, 21, 10, 0, tzinfo=UTC), + end_utc=datetime(2026, 9, 21, 11, 0, tzinfo=UTC), + organizer_email="owner@example.com", + attendees=["guest@example.com"], + ) + msg.add_alternative(ics, subtype="calendar", params={"method": "REQUEST"}) + result = detect_calendar_parts(msg.as_bytes()) + self.assertTrue(result["calendar_candidate"]) + self.assertTrue(result["is_invitation"]) + self.assertFalse(result["network_access"]) + self.assertFalse(result["external_actions"]) + + def test_cancel_is_invitation_semantics_without_external_action(self): + ics = "BEGIN:VCALENDAR\r\nMETHOD:CANCEL\r\nBEGIN:VEVENT\r\nUID:x\r\nSUMMARY:Cancelled\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n" + parsed = parse_ics(ics) + self.assertTrue(parsed["is_invitation"]) + self.assertFalse(parsed["external_actions"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_email_search_v990.py b/tests/test_email_search_v990.py new file mode 100644 index 00000000..55545852 --- /dev/null +++ b/tests/test_email_search_v990.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from email.message import EmailMessage +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest + +import numpy as np + +from postmaster.email_search_v990 import HybridEmailIndex, combine_attachment_text, extract_attachment_text + + +def fake_embed(text: str): + low = text.lower() + # Deliberately tiny deterministic semantic space for unit tests. + return np.array([ + low.count("invoice") + low.count("fattura"), + low.count("meeting") + low.count("riunione"), + low.count("printer") + low.count("stampante"), + ], dtype=np.float32) + + +class EmailSearchV990Tests(unittest.TestCase): + def test_hybrid_search_body_and_attachment_text(self): + with TemporaryDirectory() as td: + index = HybridEmailIndex(Path(td) / "index.db", embed_one=fake_embed, model_id="test") + index.upsert({ + "account_id": "a", "mailbox": "INBOX", "uid": "1", "subject": "Q3 document", + "body_text": "Here is the finance document", "attachments_text": "invoice total 1200 EUR", + "date_utc": "2026-09-20T10:00:00+00:00", + }) + index.upsert({ + "account_id": "a", "mailbox": "INBOX", "uid": "2", "subject": "Team meeting", + "body_text": "Weekly meeting", "date_utc": "2026-09-20T11:00:00+00:00", + }) + result = index.search("fattura", account_id="a", since_days=None) + self.assertEqual(result["results"][0]["uid"], "1") + self.assertTrue(result["semantic_active"]) + self.assertTrue(result["results"][0]["attachment_text_indexed"]) + + def test_local_attachment_extraction_never_fetches_network(self): + msg = EmailMessage() + msg["Subject"] = "Files" + msg.set_content("body") + msg.add_attachment(b"hello attachment", maintype="text", subtype="plain", filename="notes.txt") + extracted = extract_attachment_text(msg.as_bytes()) + self.assertEqual(extracted[0]["text"], "hello attachment") + self.assertFalse(extracted[0]["network_access"]) + combined, metadata = combine_attachment_text(extracted) + self.assertIn("hello attachment", combined) + self.assertEqual(metadata[0]["filename"], "notes.txt") + + def test_scope_is_account_bound(self): + with TemporaryDirectory() as td: + index = HybridEmailIndex(Path(td) / "index.db", embed_one=fake_embed, model_id="test") + index.upsert({"account_id": "a", "mailbox": "INBOX", "uid": "1", "subject": "invoice"}) + index.upsert({"account_id": "b", "mailbox": "INBOX", "uid": "1", "subject": "invoice secret"}) + result = index.search("invoice", account_id="a", since_days=None) + self.assertEqual({row["account_id"] for row in result["results"]}, {"a"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runtime_v990.py b/tests/test_runtime_v990.py new file mode 100644 index 00000000..63e0077f --- /dev/null +++ b/tests/test_runtime_v990.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +from pathlib import Path +from email.message import EmailMessage +from datetime import UTC, datetime +import base64 +import sys +import types + +# Local candidate tests may run without the production MCP SDK. Never shadow a real SDK: +# the full repository suite imports mcp.server/Client/CallToolResult in the same process. +try: + from mcp.types import ToolAnnotations # noqa: F401 +except (ImportError, ModuleNotFoundError): + mcp_mod = types.ModuleType("mcp") + mcp_types = types.ModuleType("mcp.types") + class ToolAnnotations: + def __init__(self, **kwargs): self.__dict__.update(kwargs) + mcp_types.ToolAnnotations = ToolAnnotations + sys.modules.setdefault("mcp", mcp_mod) + sys.modules.setdefault("mcp.types", mcp_types) +from tempfile import TemporaryDirectory +from types import SimpleNamespace +import unittest + +from postmaster.runtime_v990 import MCP_COMMAND_COUNT_V990, install_runtime_v990 +from postmaster.calendar_v990 import build_calendar_request + + +class FakeMCP: + def __init__(self): + self.tools = {} + def remove_tool(self, name): + self.tools.pop(name, None) + def add_tool(self, fn, name=None, annotations=None): + self.tools[name or fn.__name__] = (fn, annotations) + + +class FakeAccountStore: + def __init__(self, tracking_default=False): + self.tracking_default = tracking_default + def get_account(self, account_id=None): + return {"id": account_id or "acct", "tracking_default": self.tracking_default, "email_address": "me@example.com", "label": "Me"} + + +class FakeIndexEngine: + pass + + +class FakeCore(SimpleNamespace): + pass + + +class RuntimeV990Tests(unittest.TestCase): + def make(self, td, *, tracking_default=False): + mcp = FakeMCP() + calls = [] + def sent(**kwargs): + calls.append(("send", kwargs)); return {"ok": True, "sent": True} + def threaded(name): + def fn(**kwargs): calls.append((name, kwargs)); return {"ok": True, "sent": True} + return fn + ics = build_calendar_request( + uid="mail-invite@example.com", summary="Inbound invite", + start_utc=datetime(2026, 9, 22, 10, 0, tzinfo=UTC), + end_utc=datetime(2026, 9, 22, 10, 30, tzinfo=UTC), + organizer_email="sender@example.com", attendees=["me@example.com"], + video_url="https://meet.google.com/abc-defg-hij", + ) + msg = EmailMessage() + msg["Subject"] = "Invite" + msg["From"] = "sender@example.com" + msg["To"] = "me@example.com" + msg.set_content("See invite") + msg.add_alternative(ics, subtype="calendar", params={"method":"REQUEST"}) + raw_b64 = base64.b64encode(msg.as_bytes()).decode("ascii") + core = FakeCore( + mcp=mcp, + send_email=sent, + reply_email=threaded("reply"), + follow_up_email=threaded("follow"), + list_open_events=lambda **kw: {"ok": True, "events": [{"delivery_id":"d1","opened_at":"2026-09-20T10:01:01Z","user_agent":"GoogleImageProxy","client_source":"gmail_image_proxy"}]}, + list_tracking_deliveries=lambda **kw: {"ok": True, "deliveries": [{"delivery_id":"d1","sent_at":"2026-09-20T10:00:00Z"}]}, + search_emails=lambda **kw: [], + get_email=lambda **kw: {"ok": True, "email": {"uid": str(kw.get("uid") or "1"), "subject":"Invite", "raw": raw_b64}}, + list_email_attachments=lambda **kw: {"attachments": []}, + read_email_attachment=lambda **kw: {}, + get_email_attachment=lambda **kw: {}, + get_job=lambda **kw: {"id":"j1","title":"Call","description":"Discuss","schedule_type":"once","schedule_value":"2026-09-21T10:00:00+00:00","timezone":"UTC","payload":{"calendar":{"attendees":["you@example.com"],"duration_minutes":30,"video_url":"https://meet.google.com/job-call"}}}, + ) + store = FakeAccountStore(tracking_default) + base = SimpleNamespace(account_store=lambda: store, context_engine=lambda: FakeIndexEngine()) + for name in ("search_emails","get_email","list_email_attachments","read_email_attachment","get_email_attachment"): + setattr(base, name, getattr(core, name)) + result = install_runtime_v990(base, core, lambda: {"ok": True, "mcp_command_count_expected":118}, tracking_approval_db=str(Path(td)/"approval.db"), email_search_db=str(Path(td)/"search.db")) + return base, core, calls, result + + def test_effective_off_blocks_then_exact_one_use_approval_sends(self): + with TemporaryDirectory() as td: + base, core, calls, _ = self.make(td, tracking_default=False) + first = core.send_email(to=["you@example.com"], subject="Hello", body="Body") + self.assertTrue(first["approval_required"]) + self.assertFalse(first["send_performed"]) + self.assertEqual(calls, []) + second = core.send_email(to=["you@example.com"], subject="Hello", body="Body", tracking_disable_approval_id=first["preview_id"]) + self.assertTrue(second["sent"]) + self.assertEqual(len(calls), 1) + third = core.send_email(to=["you@example.com"], subject="Hello", body="Body", tracking_disable_approval_id=first["preview_id"]) + self.assertFalse(third["ok"]) + self.assertEqual(len(calls), 1) + + def test_default_on_sends_without_tracking_off_gate(self): + with TemporaryDirectory() as td: + _, core, calls, _ = self.make(td, tracking_default=True) + result = core.send_email(to=["you@example.com"], subject="Hello") + self.assertTrue(result["sent"]) + self.assertEqual(len(calls), 1) + + def test_open_events_are_enriched_with_60_second_policy(self): + with TemporaryDirectory() as td: + _, core, _, _ = self.make(td) + result = core.list_open_events() + cls = result["events"][0]["open_classification"] + self.assertEqual(cls["classification"], "likely_human_via_proxy") + self.assertEqual(cls["metadata_reliability"], "low") + self.assertEqual(result["classification_policy"]["google_image_proxy_human_delay_seconds"], 60) + + + def test_get_email_always_adds_local_calendar_analysis(self): + with TemporaryDirectory() as td: + _, core, _, _ = self.make(td, tracking_default=True) + result = core.get_email(mailbox="INBOX", uid="1") + self.assertTrue(result["calendar_analysis_policy"]["mandatory"]) + self.assertFalse(result["calendar_analysis_policy"]["external_actions"]) + analysis = result["email"]["calendar_analysis"] + self.assertTrue(analysis.get("calendar_candidate") or analysis.get("calendar_detected")) + self.assertFalse(analysis["network_access"]) + + def test_open_event_resolves_sent_at_from_delivery(self): + with TemporaryDirectory() as td: + _, core, _, _ = self.make(td) + result = core.list_open_events() + cls = result["events"][0]["open_classification"] + self.assertEqual(cls["classification"], "likely_human_via_proxy") + self.assertEqual(cls["delay_seconds"], 61.0) + + def test_calendar_invite_is_explicit_send_and_preserves_video_link(self): + with TemporaryDirectory() as td: + _, core, calls, _ = self.make(td, tracking_default=True) + result = core.send_job_calendar_invite(job_id="j1") + self.assertTrue(result["sent"]) + self.assertEqual(len(calls), 1) + payload = calls[0][1] + self.assertEqual(payload["to"], ["you@example.com"]) + self.assertEqual(payload["attachments"][0]["filename"], "invite.ics") + decoded = base64.b64decode(payload["attachments"][0]["content_base64"]).decode("utf-8") + self.assertIn("METHOD:REQUEST", decoded) + self.assertIn("https://meet.google.com/job-call", decoded) + + def test_runtime_status_declares_release_policy(self): + with TemporaryDirectory() as td: + _, core, _, result = self.make(td) + status = core.runtime_status() + self.assertEqual(status["version_capability"], "9.9.0") + self.assertEqual(status["mcp_command_count_expected"], MCP_COMMAND_COUNT_V990) + self.assertTrue(status["tracking_disable_requires_fresh_chat_approval"]) + self.assertTrue(status["stored_file_link_preferred_over_attachment"]) + + def test_new_tool_descriptions_contain_storage_and_approval_rules(self): + with TemporaryDirectory() as td: + _, core, _, _ = self.make(td) + self.assertIn("Stored File", core.send_email.__doc__) + self.assertIn("fresh explicit chat approval", core.send_email.__doc__) + self.assertIn("ASCII", core.send_email.__doc__) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_tracking_policy_v990.py b/tests/test_tracking_policy_v990.py new file mode 100644 index 00000000..8998475d --- /dev/null +++ b/tests/test_tracking_policy_v990.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest + +from postmaster.tracking_policy_v990 import ( + TrackingApprovalError, + TrackingApprovalStore, + classify_open_event, + effective_tracking, + generated_subject_guidance, +) + + +class TrackingPolicyV990Tests(unittest.TestCase): + def test_effective_tracking_respects_account_default(self): + self.assertFalse(effective_tracking(None, account_tracking_default=False)) + self.assertTrue(effective_tracking(None, account_tracking_default=True)) + self.assertFalse(effective_tracking(False, account_tracking_default=True)) + self.assertTrue(effective_tracking(True, account_tracking_default=False)) + + def test_approval_is_exact_and_one_use(self): + with TemporaryDirectory() as td: + store = TrackingApprovalStore(Path(td) / "approvals.db") + intent = {"operation": "send_email", "to": ["a@example.com"], "subject": "Hello", "body_sha256": "abc"} + preview = store.create(operation="send_email", intent=intent, summary={"to": ["a@example.com"]}) + consumed = store.consume(preview["preview_id"], operation="send_email", intent=intent) + self.assertTrue(consumed["consumed"]) + with self.assertRaises(TrackingApprovalError): + store.consume(preview["preview_id"], operation="send_email", intent=intent) + + def test_approval_rejects_mutated_intent(self): + with TemporaryDirectory() as td: + store = TrackingApprovalStore(Path(td) / "approvals.db") + preview = store.create(operation="send_email", intent={"to": ["a@example.com"]}) + with self.assertRaises(TrackingApprovalError): + store.consume(preview["preview_id"], operation="send_email", intent={"to": ["b@example.com"]}) + + def test_google_proxy_after_one_minute_is_likely_human_but_metadata_low(self): + sent = datetime(2026, 9, 20, 10, 0, tzinfo=UTC) + result = classify_open_event( + sent_at=sent, + opened_at=sent + timedelta(seconds=61), + user_agent="Mozilla/5.0 GoogleImageProxy", + client_source="gmail_image_proxy", + ) + self.assertEqual(result["classification"], "likely_human_via_proxy") + self.assertEqual(result["metadata_reliability"], "low") + self.assertEqual(result["threshold_seconds"], 60) + + def test_google_proxy_at_one_minute_is_not_promoted(self): + sent = datetime(2026, 9, 20, 10, 0, tzinfo=UTC) + result = classify_open_event(sent_at=sent, opened_at=sent + timedelta(seconds=60), user_agent="GoogleImageProxy") + self.assertEqual(result["classification"], "likely_machine_or_proxy") + + def test_ascii_is_preference_not_rewrite(self): + result = generated_subject_guidance("Ciao — prova", explicitly_user_supplied=True) + self.assertEqual(result["subject"], "Ciao — prova") + self.assertFalse(result["should_suggest_ascii"]) + self.assertTrue(result["preserve_user_supplied_unicode"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_v9_6_0_release_contracts.py b/tests/test_v9_6_0_release_contracts.py index 45923d95..96b3046e 100644 --- a/tests/test_v9_6_0_release_contracts.py +++ b/tests/test_v9_6_0_release_contracts.py @@ -50,6 +50,21 @@ "privacy_proxy_provisioning_execute", } +V990_MCP_NAMES = { + "email_search_status", + "search_emails_hybrid", + "get_email_calendar_invites", + "send_job_calendar_invite", + "whatsapp_status", + "whatsapp_start_pairing", + "whatsapp_reconnect", + "whatsapp_list_messages", + "whatsapp_send_text", + "whatsapp_send_media", + "whatsapp_list_groups", + "whatsapp_list_receipts", +} + class CanonicalUnsubscribeUrlV960Tests(unittest.TestCase): def test_public_email_base_url_then_public_mcp_host_fallback(self): @@ -255,10 +270,11 @@ def test_composed_runtime_keeps_exact_90_baseline_tool_names(self): legacy_extensions = V967_LIFECYCLE_MCP_NAMES | {"fetch_email_remote_content"} self.assertTrue(BASELINE_MCP_NAMES <= actual) self.assertTrue(legacy_extensions <= actual) - structured_extensions = actual - BASELINE_MCP_NAMES - legacy_extensions + self.assertTrue(V990_MCP_NAMES <= actual) + structured_extensions = actual - BASELINE_MCP_NAMES - legacy_extensions - V990_MCP_NAMES self.assertEqual(len(structured_extensions), 21) self.assertTrue(all(name.startswith("db_") for name in structured_extensions)) - self.assertEqual(len(actual), 118) + self.assertEqual(len(actual), 130) if __name__ == "__main__": diff --git a/tests/test_v9_6_6_privacy_provisioning.py b/tests/test_v9_6_6_privacy_provisioning.py index 2730e060..75bd2a8f 100644 --- a/tests/test_v9_6_6_privacy_provisioning.py +++ b/tests/test_v9_6_6_privacy_provisioning.py @@ -272,7 +272,21 @@ def test_final_registry_preserves_v966_legacy_privacy_schema_inside_v967_surface tools = asyncio.run(runtime.mcp.list_tools()) by_name = {tool.name: tool for tool in tools} - self.assertEqual(len(by_name), 118) + self.assertEqual(len(by_name), 130) + self.assertTrue({ + "email_search_status", + "search_emails_hybrid", + "get_email_calendar_invites", + "send_job_calendar_invite", + "whatsapp_status", + "whatsapp_start_pairing", + "whatsapp_reconnect", + "whatsapp_list_messages", + "whatsapp_send_text", + "whatsapp_send_media", + "whatsapp_list_groups", + "whatsapp_list_receipts", +} <= set(by_name)) props = by_name["set_amp_account_state"].input_schema["properties"] self.assertIn("privacy_proxy_action", props) self.assertIn("privacy_proxy_confirm", props) diff --git a/tests/test_v9_6_7_mcp_lifecycle.py b/tests/test_v9_6_7_mcp_lifecycle.py index b56a8b72..1a369ca3 100644 --- a/tests/test_v9_6_7_mcp_lifecycle.py +++ b/tests/test_v9_6_7_mcp_lifecycle.py @@ -490,7 +490,21 @@ def test_composed_runtime_has_v967_plus_one_names_and_six_new_lifecycle_commands tools = asyncio.run(runtime.mcp.list_tools()) by_name = {tool.name: tool for tool in tools} - self.assertEqual(len(by_name), 118) + self.assertEqual(len(by_name), 130) + self.assertTrue({ + "email_search_status", + "search_emails_hybrid", + "get_email_calendar_invites", + "send_job_calendar_invite", + "whatsapp_status", + "whatsapp_start_pairing", + "whatsapp_reconnect", + "whatsapp_list_messages", + "whatsapp_send_text", + "whatsapp_send_media", + "whatsapp_list_groups", + "whatsapp_list_receipts", +} <= set(by_name)) self.assertEqual(EXPECTED_V967_TOOLS - set(by_name), set()) self.assertIn("fetch_email_remote_content", by_name) self.assertIn("privacy_proxy_action", by_name["set_amp_account_state"].input_schema["properties"]) diff --git a/tests/test_v9_6_9_registry_report.py b/tests/test_v9_6_9_registry_report.py index 4562192a..bd509047 100644 --- a/tests/test_v9_6_9_registry_report.py +++ b/tests/test_v9_6_9_registry_report.py @@ -12,7 +12,21 @@ def test_final_composed_registry_count_names_and_fetch_schema(self): tools = asyncio.run(runtime.mcp.list_tools()) by_name = {tool.name: tool for tool in tools} names = sorted(by_name) - self.assertEqual(len(names), 118) + self.assertEqual(len(names), 130) + self.assertTrue({ + "email_search_status", + "search_emails_hybrid", + "get_email_calendar_invites", + "send_job_calendar_invite", + "whatsapp_status", + "whatsapp_start_pairing", + "whatsapp_reconnect", + "whatsapp_list_messages", + "whatsapp_send_text", + "whatsapp_send_media", + "whatsapp_list_groups", + "whatsapp_list_receipts", +} <= set(by_name)) self.assertIn("fetch_email_remote_content", by_name) self.assertIn("db_status", by_name) self.assertIn("db_link_memory", by_name) diff --git a/tests/test_v9_6_production_integration_audit.py b/tests/test_v9_6_production_integration_audit.py index 2086e44a..339f1094 100644 --- a/tests/test_v9_6_production_integration_audit.py +++ b/tests/test_v9_6_production_integration_audit.py @@ -151,7 +151,21 @@ def test_composed_runtime_exports_v960_v963_v964_schema_through_real_registry(se tools = asyncio.run(runtime.mcp.list_tools()) by_name = {tool.name: tool for tool in tools} - self.assertEqual(len(by_name), 118) + self.assertEqual(len(by_name), 130) + self.assertTrue({ + "email_search_status", + "search_emails_hybrid", + "get_email_calendar_invites", + "send_job_calendar_invite", + "whatsapp_status", + "whatsapp_start_pairing", + "whatsapp_reconnect", + "whatsapp_list_messages", + "whatsapp_send_text", + "whatsapp_send_media", + "whatsapp_list_groups", + "whatsapp_list_receipts", +} <= set(by_name)) self.assertIn("fetch_email_remote_content", by_name) get_email = by_name["get_email"].input_schema["properties"] diff --git a/tests/test_v9_7_2_final_mail_composition.py b/tests/test_v9_7_2_final_mail_composition.py index b0fe04f6..891b3ac5 100644 --- a/tests/test_v9_7_2_final_mail_composition.py +++ b/tests/test_v9_7_2_final_mail_composition.py @@ -20,8 +20,23 @@ def test_final_runtime_mail_factory_is_v972_and_uses_v972_core_store(self) -> No self.assertEqual(core.link_store.__name__, "_store") tools = runtime.mcp._tool_manager.list_tools() - self.assertEqual(len(tools), 118) - self.assertIn("send_email", {tool.name for tool in tools}) + self.assertEqual(len(tools), 130) + names = {tool.name for tool in tools} + self.assertTrue({ + "email_search_status", + "search_emails_hybrid", + "get_email_calendar_invites", + "send_job_calendar_invite", + "whatsapp_status", + "whatsapp_start_pairing", + "whatsapp_reconnect", + "whatsapp_list_messages", + "whatsapp_send_text", + "whatsapp_send_media", + "whatsapp_list_groups", + "whatsapp_list_receipts", +} <= names) + self.assertIn("send_email", names) self.assertIn("get_stored_file_resource", {tool.name for tool in tools}) diff --git a/tests/test_v9_7_2_stored_file_public_download.py b/tests/test_v9_7_2_stored_file_public_download.py index a24808a7..37a86f41 100644 --- a/tests/test_v9_7_2_stored_file_public_download.py +++ b/tests/test_v9_7_2_stored_file_public_download.py @@ -467,8 +467,22 @@ def test_additive_schema_final_runtime_and_public_mcp_surface(self) -> None: self._route("/t/c/{token}") tools = self.runtime.mcp._tool_manager.list_tools() - self.assertEqual(len(tools), 118) + self.assertEqual(len(tools), 130) names = {tool.name for tool in tools} + self.assertTrue({ + "email_search_status", + "search_emails_hybrid", + "get_email_calendar_invites", + "send_job_calendar_invite", + "whatsapp_status", + "whatsapp_start_pairing", + "whatsapp_reconnect", + "whatsapp_list_messages", + "whatsapp_send_text", + "whatsapp_send_media", + "whatsapp_list_groups", + "whatsapp_list_receipts", +} <= names) self.assertIn("get_stored_file_resource", names) status = self.runtime.tracking_status() diff --git a/tests/test_whatsapp_adapter_v990.py b/tests/test_whatsapp_adapter_v990.py new file mode 100644 index 00000000..98bcf390 --- /dev/null +++ b/tests/test_whatsapp_adapter_v990.py @@ -0,0 +1,320 @@ +from __future__ import annotations + +import asyncio +import base64 +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest + +from postmaster.whatsapp_v990.adapter import CurrentProtocolAdapter, ProtocolCredentials +from postmaster.whatsapp_v990.binary import BinaryNode +from postmaster.whatsapp_v990.crypto import generate_curve_keypair +from postmaster.whatsapp_v990.messages import encode_text_message, pad_random_max16 +from postmaster.whatsapp_v990.signal_keys import SignalPreKeyBundle, generate_signed_pre_key +from postmaster.whatsapp_v990.signal_session import EncryptedSignalSessionStore, initialize_outgoing_session +from postmaster.whatsapp_v990.store import EncryptedAuthStore + + +class FakeSession: + def __init__(self, nodes, query_responses=None): + self.nodes=list(nodes) + self.query_responses=list(query_responses or []) + self.queries=[] + self.sent=[] + self.closed=False + + async def recv_node(self, *, timeout=30.0): + if not self.nodes: + await asyncio.sleep(3600) + return self.nodes.pop(0) + + async def send_node(self, node): + self.sent.append(node) + + async def query(self, node, *, timeout=30.0): + self.queries.append(node) + if not self.query_responses: + raise AssertionError("unexpected query without prepared response") + return self.query_responses.pop(0) + + async def send_and_wait(self, node, *, response_tag=None, timeout=30.0): + await self.send_node(node) + response=await self.recv_node(timeout=timeout) + if response_tag is not None and response.tag != response_tag: + raise AssertionError(f"unexpected correlated response tag {response.tag}") + return response + + async def close(self): + self.closed=True + + +class WhatsAppAdapterV990Tests(unittest.IsolatedAsyncioTestCase): + def store(self, td): + return EncryptedAuthStore(str(Path(td)/"auth.db"), key_path=str(Path(td)/"auth.key")) + + async def test_construction_and_status_do_not_open_network(self): + with TemporaryDirectory() as td: + called=0 + async def opener(**kwargs): + nonlocal called + called += 1 + raise AssertionError("network/session opener must not run during construction or status") + adapter=CurrentProtocolAdapter(self.store(td), session_opener=opener) + status=adapter.status() + self.assertEqual(called,0) + self.assertTrue(status["configured"]) + self.assertFalse(status["connected"]) + self.assertFalse(status["paired"]) + self.assertFalse(status["signal_send_ready"]) + + async def test_explicit_pairing_returns_qr_and_persists_private_material_only_encrypted(self): + with TemporaryDirectory() as td: + session=FakeSession([ + BinaryNode( + "iq", + {"id":"pair-1","type":"set"}, + [BinaryNode("pair-device",{},[ + BinaryNode("ref",{},b"ref-one"), + BinaryNode("ref",{},b"ref-two"), + ])], + ) + ]) + calls=[] + async def opener(**kwargs): + calls.append(kwargs) + return session + auth=self.store(td) + adapter=CurrentProtocolAdapter(auth, session_opener=opener) + result=await adapter.start_pairing() + self.assertTrue(result["ok"]) + self.assertTrue(result["qr"].startswith("https://wa.me/settings/linked_devices#")) + self.assertNotIn("private_key",result) + self.assertFalse(result["private_material_exposed"]) + self.assertEqual(len(calls),1) + self.assertTrue(calls[0]["client_payload"]) + self.assertEqual(len(calls[0]["noise_static"].private),32) + self.assertEqual(session.sent[0].tag,"iq") + self.assertEqual(session.sent[0].attrs["type"],"result") + self.assertIsNotNone(auth.get_json("protocol","credentials")) + raw=(Path(td)/"auth.db").read_bytes() + self.assertNotIn(calls[0]["noise_static"].private,raw) + self.assertTrue(adapter.status()["pairing_pending"]) + await adapter.close() + self.assertTrue(session.closed) + + async def test_reconnect_without_pairing_fails_closed_without_network(self): + with TemporaryDirectory() as td: + calls=0 + async def opener(**kwargs): + nonlocal calls + calls += 1 + raise AssertionError("should not open a session without paired credentials") + adapter=CurrentProtocolAdapter(self.store(td), session_opener=opener) + with self.assertRaisesRegex(Exception,"No paired WhatsApp session"): + await adapter.reconnect() + self.assertEqual(calls,0) + + + def _usync_direct_result(self): + return BinaryNode("iq",{"id":"u1","type":"result"},[ + BinaryNode("usync",{},[ + BinaryNode("list",{},[ + BinaryNode("user",{"jid":"111@s.whatsapp.net"},[ + BinaryNode("lid",{"val":"999@lid"}), + BinaryNode("devices",{},[ + BinaryNode("device-list",{},[ + BinaryNode("device",{"id":"0"}), + BinaryNode("device",{"id":"2","key-index":"7"}), + ]) + ]), + ]), + BinaryNode("user",{"jid":"222@s.whatsapp.net"},[ + BinaryNode("lid",{"val":"888@lid"}), + BinaryNode("devices",{},[ + BinaryNode("device-list",{},[ + BinaryNode("device",{"id":"0"}), + BinaryNode("device",{"id":"3","key-index":"8"}), + ]) + ]), + ]), + ]) + ]) + ]) + + def _connected_direct_adapter(self, td, *, reply_ack_id=None): + auth=self.store(td) + identity=generate_curve_keypair() + creds=ProtocolCredentials( + noise=generate_curve_keypair(), + identity=identity, + signed_pre_key=generate_signed_pre_key(identity,1), + registration_id=111, + adv_secret_b64=base64.b64encode(bytes(range(32))).decode("ascii"), + registered=True, + jid="111:2@s.whatsapp.net", + lid="999:2@lid", + account_identity_b64=base64.b64encode(b"adv-device-identity").decode("ascii"), + ) + ack=BinaryNode("ack",{"id":reply_ack_id or "placeholder","class":"message"}) + session=FakeSession([ack],[self._usync_direct_result()]) + adapter=CurrentProtocolAdapter(auth) + adapter._save(creds) + adapter.session=session + adapter._connected=True + + sessions=EncryptedSignalSessionStore(auth) + for address in ("999@lid","888@lid","888:3@lid"): + remote_identity=generate_curve_keypair() + remote_signed=generate_signed_pre_key(remote_identity,17) + bundle=SignalPreKeyBundle( + registration_id=222, + identity_key=remote_identity.public, + signed_pre_key_id=remote_signed.key_id, + signed_pre_key=remote_signed.key_pair.public, + signed_pre_key_signature=remote_signed.signature, + ) + sessions.save( + address, + initialize_outgoing_session( + our_identity=creds.identity, + our_registration_id=creds.registration_id, + bundle=bundle, + ), + ) + return adapter,session + + async def test_direct_text_fans_out_to_remote_and_other_own_devices_and_waits_for_ack(self): + with TemporaryDirectory() as td: + adapter,session=self._connected_direct_adapter(td) + # Message ids are random; make the prepared ACK follow the outgoing stanza id. + original_send=session.send_node + async def send_and_ack(node): + await original_send(node) + if node.tag=="message": + session.nodes[0].attrs["id"]=node.attrs["id"] + session.send_node=send_and_ack + + result=await adapter.send_text(jid="222@s.whatsapp.net",text="hello",emit_read_receipt=False) + self.assertTrue(result["ok"]) + self.assertTrue(result["server_ack"]) + self.assertEqual(result["device_fanout"],3) + self.assertEqual(result["own_device_targets"],1) + self.assertEqual(result["remote_device_targets"],2) + self.assertTrue(result["used_prekey_message"]) + self.assertFalse(result["read_receipt_emitted"]) + + self.assertEqual(len(session.queries),1) + self.assertEqual(session.queries[0].attrs["xmlns"],"usync") + stanza=session.sent[0] + self.assertEqual(stanza.tag,"message") + self.assertEqual(stanza.attrs["to"],"222@s.whatsapp.net") + participants=stanza.child("participants").children("to") + self.assertEqual( + {node.attrs["jid"] for node in participants}, + {"999@lid","888@lid","888:3@lid"}, + ) + self.assertNotIn("999:2@lid",{node.attrs["jid"] for node in participants}) + self.assertEqual(stanza.child("device-identity").content,b"adv-device-identity") + + async def test_reply_emits_read_receipt_only_after_message_ack(self): + with TemporaryDirectory() as td: + adapter,session=self._connected_direct_adapter(td) + order=[] + original_send=session.send_node + async def send_and_ack(node): + order.append(node.tag) + await original_send(node) + if node.tag=="message": + session.nodes[0].attrs["id"]=node.attrs["id"] + session.send_node=send_and_ack + + result=await adapter.send_text( + jid="222@s.whatsapp.net", + text="reply", + reply_to_message_id="incoming-1", + emit_read_receipt=True, + ) + self.assertTrue(result["ok"]) + self.assertTrue(result["read_receipt_emitted"]) + self.assertEqual(order,["message","receipt"]) + receipt=session.sent[-1] + self.assertEqual(receipt.attrs["id"],"incoming-1") + self.assertEqual(receipt.attrs["type"],"read") + + + async def test_incoming_pkmsg_decrypts_persists_and_transport_acks_without_read_receipt(self): + with TemporaryDirectory() as td: + adapter,session=self._connected_direct_adapter(td) + creds=adapter._load() + self.assertIsNotNone(creds) + messages=[] + receipts=[] + adapter.on_message=lambda **kwargs: messages.append(kwargs) + adapter.on_receipt=lambda **kwargs: receipts.append(kwargs) + + one_time=generate_curve_keypair() + adapter._store_pre_key(7,one_time) + remote_identity=generate_curve_keypair() + bundle=SignalPreKeyBundle( + registration_id=creds.registration_id, + identity_key=creds.identity.public, + signed_pre_key_id=creds.signed_pre_key.key_id, + signed_pre_key=creds.signed_pre_key.key_pair.public, + signed_pre_key_signature=creds.signed_pre_key.signature, + pre_key_id=7, + pre_key=one_time.public, + ) + sender=initialize_outgoing_session( + our_identity=remote_identity, + our_registration_id=222, + bundle=bundle, + ) + kind,ciphertext=sender.encrypt( + pad_random_max16(encode_text_message("incoming hello"),random1=b"\x00") + ) + self.assertEqual(kind,"pkmsg") + session.sent.clear() + incoming=BinaryNode( + "message", + {"id":"incoming-1","from":"222:3@lid","type":"text","t":"1790000000"}, + [BinaryNode("enc",{"v":"2","type":"pkmsg"},ciphertext)], + ) + await adapter._handle_unsolicited(incoming) + + self.assertEqual(len(messages),1) + self.assertEqual(messages[0]["message_id"],"incoming-1") + self.assertEqual(messages[0]["jid"],"222@lid") + self.assertEqual(messages[0]["direction"],"in") + self.assertEqual(messages[0]["kind"],"text") + self.assertEqual(messages[0]["text"],"incoming hello") + self.assertEqual(receipts,[]) + self.assertIsNone(adapter._load_pre_key(7)) + self.assertEqual(len(session.sent),1) + ack=session.sent[0] + self.assertEqual(ack.tag,"ack") + self.assertEqual(ack.attrs["id"],"incoming-1") + self.assertEqual(ack.attrs["class"],"message") + self.assertEqual(ack.attrs["from"],creds.jid) + self.assertNotEqual(ack.attrs.get("type"),"read") + + async def test_remote_receipt_is_recorded_and_transport_acked(self): + with TemporaryDirectory() as td: + adapter,session=self._connected_direct_adapter(td) + receipts=[] + adapter.on_receipt=lambda **kwargs: receipts.append(kwargs) + session.sent.clear() + node=BinaryNode("receipt",{"id":"sent-1","from":"222@s.whatsapp.net","type":"read"}) + await adapter._handle_unsolicited(node) + self.assertEqual(receipts,[{ + "message_id":"sent-1", + "jid":"222@s.whatsapp.net", + "receipt_type":"read", + "source":"remote", + }]) + self.assertEqual(len(session.sent),1) + self.assertEqual(session.sent[0].tag,"ack") + self.assertEqual(session.sent[0].attrs["class"],"receipt") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_whatsapp_adv_v990.py b/tests/test_whatsapp_adv_v990.py new file mode 100644 index 00000000..fdb5227c --- /dev/null +++ b/tests/test_whatsapp_adv_v990.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import hashlib +import hmac +import unittest + +from postmaster.whatsapp_v990.adv import ( + ADVDeviceIdentity, + ADVSignedDeviceIdentity, + ADVSignedDeviceIdentityHMAC, + ADV_E2EE, + ADVError, + WA_ADV_ACCOUNT_SIG_PREFIX, + WA_ADV_DEVICE_SIG_PREFIX, + decode_signed_device_identity, + encode_adv_device_identity, + encode_signed_device_identity, + encode_signed_device_identity_hmac, + verify_and_sign_pair_success_identity, +) +from postmaster.whatsapp_v990.crypto import generate_curve_keypair, xeddsa_sign, xeddsa_verify + + +class WhatsAppAdvV990Tests(unittest.TestCase): + def build_fixture(self): + signed_identity = generate_curve_keypair() + account_key = generate_curve_keypair() + device_details = encode_adv_device_identity( + ADVDeviceIdentity(raw_id=11, timestamp=1720000000, key_index=7, account_type=ADV_E2EE, device_type=ADV_E2EE) + ) + account_message = WA_ADV_ACCOUNT_SIG_PREFIX + device_details + signed_identity.public + account_signature = xeddsa_sign(account_key.private, account_message, random64=b"a" * 64) + account = ADVSignedDeviceIdentity( + details=device_details, + account_signature_key=account_key.public, + account_signature=account_signature, + ) + account_blob = encode_signed_device_identity(account, include_signature_key=True) + adv_secret = bytes(range(32)) + mac = hmac.new(adv_secret, account_blob, hashlib.sha256).digest() + outer = encode_signed_device_identity_hmac( + ADVSignedDeviceIdentityHMAC(details=account_blob, hmac_value=mac, account_type=ADV_E2EE) + ) + return signed_identity, account_key, adv_secret, device_details, outer + + def test_pair_success_hmac_account_and_device_signatures(self): + signed_identity, account_key, adv_secret, device_details, outer = self.build_fixture() + verified = verify_and_sign_pair_success_identity( + outer, + adv_secret_key=adv_secret, + signed_identity_key=signed_identity, + random64=b"b" * 64, + ) + self.assertEqual(verified.device_identity.key_index, 7) + self.assertTrue(verified.hmac_verified) + self.assertTrue(verified.account_signature_verified) + reply = decode_signed_device_identity(verified.encoded_for_reply) + self.assertIsNone(reply.account_signature_key) + self.assertIsNotNone(reply.device_signature) + device_message = ( + WA_ADV_DEVICE_SIG_PREFIX + + device_details + + signed_identity.public + + account_key.public + ) + self.assertTrue(xeddsa_verify(signed_identity.public, device_message, reply.device_signature)) + + def test_tampered_hmac_fails_closed(self): + signed_identity, _account_key, adv_secret, _details, outer = self.build_fixture() + tampered = bytearray(outer) + tampered[-1] ^= 1 + with self.assertRaises(ADVError): + verify_and_sign_pair_success_identity( + bytes(tampered), + adv_secret_key=adv_secret, + signed_identity_key=signed_identity, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_whatsapp_cert_v990.py b/tests/test_whatsapp_cert_v990.py new file mode 100644 index 00000000..2b9cc3b5 --- /dev/null +++ b/tests/test_whatsapp_cert_v990.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import unittest + +from postmaster.whatsapp_v990.cert import NoiseCertificateError, verify_noise_certificate_chain +from postmaster.whatsapp_v990.crypto import generate_curve_keypair, xeddsa_sign +from postmaster.whatsapp_v990.proto import field_bytes, field_message, field_varint + + +def _details(*, serial: int, issuer: int, key: bytes, not_before: int, not_after: int) -> bytes: + return b"".join( + ( + field_varint(1, serial), + field_varint(2, issuer), + field_bytes(3, key), + field_varint(4, not_before), + field_varint(5, not_after), + ) + ) + + +def _cert(details: bytes, signature: bytes) -> bytes: + return field_bytes(1, details) + field_bytes(2, signature) + + +class WhatsAppNoiseCertificateV990Tests(unittest.TestCase): + def build_chain(self): + now = 1_800_000_000 + root = generate_curve_keypair() + intermediate = generate_curve_keypair() + leaf = generate_curve_keypair() + + intermediate_details = _details( + serial=1, issuer=0, key=intermediate.public, not_before=now - 60, not_after=now + 60 + ) + intermediate_signature = xeddsa_sign( + root.private, intermediate_details, random64=b"r" * 64 + ) + leaf_details = _details( + serial=2, issuer=1, key=leaf.public, not_before=now - 60, not_after=now + 60 + ) + leaf_signature = xeddsa_sign( + intermediate.private, leaf_details, random64=b"i" * 64 + ) + chain = field_message(1, [_cert(leaf_details, leaf_signature)]) + field_message( + 2, [_cert(intermediate_details, intermediate_signature)] + ) + return now, root, chain + + def test_valid_chain_verifies_to_injected_root(self): + now, root, chain = self.build_chain() + verified = verify_noise_certificate_chain( + chain, root_public_key=root.public, root_serial=0, now=now + ) + self.assertTrue(verified.chain_signatures_verified) + self.assertEqual(verified.intermediate.issuer_serial, 0) + self.assertEqual(verified.intermediate.serial, 1) + self.assertEqual(verified.leaf.serial, 2) + + def test_tampered_chain_fails_closed(self): + now, root, chain = self.build_chain() + tampered = bytearray(chain) + tampered[-1] ^= 1 + with self.assertRaises(NoiseCertificateError): + verify_noise_certificate_chain( + bytes(tampered), root_public_key=root.public, root_serial=0, now=now + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_whatsapp_client_payload_v990.py b/tests/test_whatsapp_client_payload_v990.py new file mode 100644 index 00000000..e018cd70 --- /dev/null +++ b/tests/test_whatsapp_client_payload_v990.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import hashlib +import unittest + +from postmaster.whatsapp_v990.client_payload import ( + CURRENT_WA_WEB_VERSION, + DEVICE_PROPS_VERSION, + RegistrationKeys, + build_login_payload, + build_registration_payload, + parse_web_version, +) +from postmaster.whatsapp_v990.crypto import generate_curve_keypair +from postmaster.whatsapp_v990.proto import decode_fields +from postmaster.whatsapp_v990.signal_keys import generate_signed_pre_key + + +def fmap(raw: bytes): + result = {} + for field in decode_fields(raw): + result.setdefault(field.number, []).append(field.value) + return result + + +class WhatsAppClientPayloadV990Tests(unittest.TestCase): + def test_current_version_and_parser(self): + self.assertEqual(CURRENT_WA_WEB_VERSION, (2, 3000, 1048032155)) + self.assertEqual(parse_web_version("2.3000.1048032155-alpha"), CURRENT_WA_WEB_VERSION) + self.assertEqual(parse_web_version([2, 3000, 42]), (2, 3000, 42)) + + def test_registration_payload_uses_current_public_wire_fields(self): + identity = generate_curve_keypair() + signed = generate_signed_pre_key(identity, 1, random64=b"r" * 64) + raw = build_registration_payload( + RegistrationKeys(94, identity.public, signed), + version=CURRENT_WA_WEB_VERSION, + os_name="Linux", + ) + top = fmap(raw) + self.assertEqual(top[3][-1], 0) + self.assertEqual(top[12][-1], 1) + self.assertEqual(top[13][-1], 1) + self.assertEqual(top[33][-1], 0) + + pairing = fmap(top[19][-1]) + self.assertEqual(pairing[1][-1], (94).to_bytes(4, "big")) + self.assertEqual(pairing[2][-1], b"\x05") + self.assertEqual(pairing[3][-1], identity.public) + self.assertEqual(pairing[4][-1], (1).to_bytes(3, "big")) + self.assertEqual(pairing[5][-1], signed.key_pair.public) + self.assertEqual(pairing[6][-1], signed.signature) + self.assertEqual( + pairing[7][-1], + hashlib.md5(b"2.3000.1048032155", usedforsecurity=False).digest(), + ) + + props = fmap(pairing[8][-1]) + self.assertEqual(props[1][-1], b"Linux") + self.assertEqual(props[3][-1], 1) + self.assertEqual(props[4][-1], 0) + version = fmap(props[2][-1]) + self.assertEqual((version[1][-1], version[2][-1], version[3][-1]), DEVICE_PROPS_VERSION) + + def test_login_payload_fields(self): + raw = build_login_payload(username=12345, device=7) + top = fmap(raw) + self.assertEqual(top[1][-1], 12345) + self.assertEqual(top[3][-1], 1) + self.assertEqual(top[18][-1], 7) + self.assertEqual(top[33][-1], 1) + self.assertEqual(top[41][-1], 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_whatsapp_controlled_harness_v990.py b/tests/test_whatsapp_controlled_harness_v990.py new file mode 100644 index 00000000..91972c54 --- /dev/null +++ b/tests/test_whatsapp_controlled_harness_v990.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from dataclasses import asdict +import importlib.util +from pathlib import Path +import sys +from tempfile import TemporaryDirectory +import unittest + + +def _load_harness(): + root = Path(__file__).resolve().parents[1] + path = root / "scripts" / "whatsapp_v990_controlled_interop.py" + spec = importlib.util.spec_from_file_location("whatsapp_v990_controlled_interop_test", path) + if spec is None or spec.loader is None: + raise RuntimeError("Unable to load controlled WhatsApp interop harness") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class ControlledInteropHarnessV990Tests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.harness = _load_harness() + + def test_evidence_schema_contains_only_non_secret_release_signals(self): + evidence = asdict(self.harness.Evidence()) + self.assertIn("controlled_account_interop_observed", evidence) + self.assertIn("group_sender_key_interop_observed", evidence) + self.assertIn("media_interop_observed", evidence) + self.assertIn("receipt_policy_observed", evidence) + forbidden_fragments = ( + "jid", + "qr", + "auth", + "private", + "secret", + "token", + "payload", + "filename", + "phone", + ) + for key in evidence: + lowered = key.lower() + self.assertFalse( + any(fragment in lowered for fragment in forbidden_fragments), + f"Evidence field must not expose controlled-account identifiers or secrets: {key}", + ) + + def test_harness_file_store_keeps_payloads_in_memory_and_counts_inbound(self): + store = self.harness.HarnessFileStore() + info, outbound = store.raw_bytes("controlled-interop-document") + self.assertEqual(info["media_type"], "text/plain") + self.assertIn(b"controlled WhatsApp", outbound) + + saved = store.save_bytes( + owner_id="controlled-interop", + filename="peer.txt", + data=b"inbound", + media_type="text/plain", + tags=["whatsapp", "inbound"], + ) + self.assertEqual(store.inbound_saved, 1) + saved_info, saved_bytes = store.raw_bytes(saved["id"]) + self.assertEqual(saved_info["size_bytes"], 7) + self.assertEqual(saved_bytes, b"inbound") + + def test_secret_file_writer_creates_owner_only_permissions(self): + with TemporaryDirectory() as td: + path = Path(td) / "auth.key" + self.harness._write_secret(path, b"k" * 32) + self.assertEqual(path.read_bytes(), b"k" * 32) + self.assertEqual(path.stat().st_mode & 0o077, 0) + + def test_full_acceptance_defaults_require_every_remote_observation(self): + self.assertTrue(self.harness._bool("__POSTMASTER_TEST_UNSET_TRUE__", True)) + self.assertFalse(self.harness._bool("__POSTMASTER_TEST_UNSET_FALSE__", False)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_whatsapp_crypto_v990.py b/tests/test_whatsapp_crypto_v990.py new file mode 100644 index 00000000..c280164b --- /dev/null +++ b/tests/test_whatsapp_crypto_v990.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import os +import unittest + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + +from postmaster.whatsapp_v990.crypto import ( + NOISE_WA_HEADER, + WhatsAppNoiseXX, + curve_shared_key, + decrypt_media, + encrypt_media, + frame_noise_payload, + generate_curve_keypair, + split_noise_frames, + xeddsa_public_edwards, + xeddsa_sign, + xeddsa_verify, +) + + +class WhatsAppCryptoV990Tests(unittest.TestCase): + def test_curve_shared_secret_matches(self): + a = generate_curve_keypair(); b = generate_curve_keypair() + self.assertEqual(curve_shared_key(a.private, b.public), curve_shared_key(b.private, a.public)) + + def test_xeddsa_signature_self_and_ed25519_verification(self): + pair = generate_curve_keypair() + msg = b"postmaster xeddsa compatibility" + sig = xeddsa_sign(pair.private, msg, random64=bytes(range(64))) + self.assertTrue(xeddsa_verify(pair.public, msg, sig)) + self.assertFalse(xeddsa_verify(pair.public, msg + b"!", sig)) + Ed25519PublicKey.from_public_bytes(xeddsa_public_edwards(pair.public)).verify(sig, msg) + + def test_media_roundtrip_and_tamper_rejection(self): + plaintext = os.urandom(913) + b"document" + enc = encrypt_media(plaintext, "document", media_key=bytes(range(32))) + self.assertEqual(decrypt_media(enc["encrypted"], "document", enc["media_key"], expected_file_sha256=enc["file_sha256"]), plaintext) + tampered = bytearray(enc["encrypted"]); tampered[-1] ^= 1 + with self.assertRaises(ValueError): + decrypt_media(bytes(tampered), "document", enc["media_key"]) + + def test_noise_frame_split_handles_partial_data(self): + payloads = [b"one", os.urandom(100), b""] + stream = b"".join(frame_noise_payload(p) for p in payloads) + frames, rest = split_noise_frames(stream[:-7]) + self.assertEqual(frames, payloads[:1]) + more, rest2 = split_noise_frames(rest + stream[-7:]) + self.assertEqual(more, payloads[1:]) + self.assertEqual(rest2, b"") + + def test_noise_symmetric_handshake_steps_are_deterministic(self): + # Full WA server certificate acceptance belongs to protocol tests. Here we validate + # that identical state transitions stay synchronized for encrypt/decrypt directions. + a = generate_curve_keypair() + n = WhatsAppNoiseXX(a, header=NOISE_WA_HEADER) + self.assertEqual(len(n.hash), 32) + self.assertEqual(n.counter, 0) + + +if __name__ == "__main__": unittest.main() diff --git a/tests/test_whatsapp_groups_v990.py b/tests/test_whatsapp_groups_v990.py new file mode 100644 index 00000000..4a174169 --- /dev/null +++ b/tests/test_whatsapp_groups_v990.py @@ -0,0 +1,589 @@ +from __future__ import annotations + +from pathlib import Path +from tempfile import TemporaryDirectory +import types +import unittest +from unittest.mock import AsyncMock, patch + +from postmaster.whatsapp_v990.adapter import CurrentProtocolAdapter, ProtocolCredentials +from postmaster.whatsapp_v990.binary import BinaryNode +from postmaster.whatsapp_v990.crypto import generate_curve_keypair +from postmaster.whatsapp_v990.groups import ( + WhatsAppGroupError, + build_group_metadata_query, + build_participating_groups_query, + parse_group_metadata, + parse_participating_groups, +) +from postmaster.whatsapp_v990.media import MediaUpload, encode_media_message +from postmaster.whatsapp_v990.messages import encode_text_message, pad_random_max16 +from postmaster.whatsapp_v990.sender_key import ( + EncryptedSenderKeyStore, + encode_sender_key_distribution_message, +) +from postmaster.whatsapp_v990.signal_keys import generate_registration_id, generate_signed_pre_key +from postmaster.whatsapp_v990.store import EncryptedAuthStore + + +class FakeWire: + def __init__(self, response: BinaryNode): + self.response = response + self.closed = False + self.queries: list[BinaryNode] = [] + + async def query(self, node: BinaryNode, *, timeout: float = 30.0) -> BinaryNode: + self.queries.append(node) + return self.response + + +def group_node() -> BinaryNode: + return BinaryNode( + "group", + { + "id": "12345", + "subject": "Postmaster test", + "size": "2", + "creation": "1700000000", + "creator": "111@s.whatsapp.net", + "addressing_mode": "lid", + }, + [ + BinaryNode("participant", {"jid": "111@s.whatsapp.net", "type": "admin", "lid": "900@lid"}), + BinaryNode("participant", {"jid": "222@s.whatsapp.net"}), + BinaryNode( + "description", + {"id": "d1", "participant": "111@s.whatsapp.net", "t": "1700000100"}, + [BinaryNode("body", {}, b"hello group")], + ), + BinaryNode("announcement"), + BinaryNode("ephemeral", {"expiration": "86400"}), + ], + ) + + +class WhatsAppGroupsV990Tests(unittest.IsolatedAsyncioTestCase): + def test_participating_query_matches_current_wg2_shape(self): + node = build_participating_groups_query() + self.assertEqual(node.tag, "iq") + self.assertEqual(node.attrs, {"to": "@g.us", "xmlns": "w:g2", "type": "get"}) + participating = node.child("participating") + self.assertIsNotNone(participating) + self.assertEqual([child.tag for child in participating.children()], ["participants", "description"]) + + def test_group_metadata_parser_preserves_participants_and_flags(self): + parsed = parse_group_metadata(group_node()) + self.assertEqual(parsed["id"], "12345@g.us") + self.assertEqual(parsed["subject"], "Postmaster test") + self.assertEqual(parsed["addressing_mode"], "lid") + self.assertEqual(parsed["description"], "hello group") + self.assertTrue(parsed["announce"]) + self.assertEqual(parsed["ephemeral_duration"], 86400) + self.assertEqual(len(parsed["participants"]), 2) + self.assertEqual(parsed["participants"][0]["admin"], "admin") + self.assertEqual(parsed["participants"][0]["lid"], "900@lid") + + def test_group_metadata_query_requires_group_jid(self): + with self.assertRaises(WhatsAppGroupError): + build_group_metadata_query("123@s.whatsapp.net") + node = build_group_metadata_query("123@g.us") + self.assertEqual(node.attrs["to"], "123@g.us") + self.assertEqual(node.child("query").attrs["request"], "interactive") + + def test_participating_parser_surfaces_server_error(self): + response = BinaryNode("iq", {"type": "error"}, [BinaryNode("error", {"code": "403", "text": "forbidden"})]) + with self.assertRaises(WhatsAppGroupError): + parse_participating_groups(response) + + async def test_group_text_send_distributes_sender_key_once_then_reuses_it(self): + metadata_response = BinaryNode( + "iq", + {"type": "result"}, + [ + BinaryNode( + "group", + {"id": "12345", "subject": "Test", "addressing_mode": "pn"}, + [ + BinaryNode("participant", {"jid": "111@s.whatsapp.net"}), + BinaryNode("participant", {"jid": "222@s.whatsapp.net"}), + ], + ) + ], + ) + usync_response = BinaryNode( + "iq", + {"type": "result"}, + [ + BinaryNode( + "usync", + {}, + [ + BinaryNode( + "list", + {}, + [ + BinaryNode( + "user", + {"jid": "111@s.whatsapp.net"}, + [ + BinaryNode( + "devices", + {}, + [ + BinaryNode( + "device-list", + {}, + [ + BinaryNode("device", {"id": "1", "key-index": "7"}), + BinaryNode("device", {"id": "7", "key-index": "9"}), + ], + ) + ], + ) + ], + ), + BinaryNode( + "user", + {"jid": "222@s.whatsapp.net"}, + [ + BinaryNode( + "devices", + {}, + [ + BinaryNode( + "device-list", + {}, + [BinaryNode("device", {"id": "0"})], + ) + ], + ) + ], + ), + ], + ) + ], + ) + ], + ) + + class Signal: + def encrypt(self, data): + self.last = bytes(data) + return "msg", b"pairwise-" + bytes(data[:4]) + + class Sessions: + def __init__(self, jids): + self.items = {jid: Signal() for jid in jids} + def load(self, jid): + return self.items.get(jid) + def save(self, jid, signal): + self.items[jid] = signal + + class GroupWire: + def __init__(self): + self.closed = False + self.queries = [] + self.sent = [] + async def query(self, node, *, timeout=30.0): + self.queries.append(node) + if node.attrs.get("xmlns") == "w:g2": + return metadata_response + if node.attrs.get("xmlns") == "usync": + return usync_response + raise AssertionError(f"unexpected query {node.attrs}") + async def send_and_wait(self, node, *, response_tag=None, timeout=30.0): + self.sent.append(node) + return BinaryNode("ack", {"id": node.attrs["id"], "class": "message"}) + + with TemporaryDirectory() as td: + auth = EncryptedAuthStore(str(Path(td) / "auth.db")) + adapter = CurrentProtocolAdapter(auth) + identity = generate_curve_keypair() + creds = ProtocolCredentials( + noise=generate_curve_keypair(), + identity=identity, + signed_pre_key=generate_signed_pre_key(identity, 1), + registration_id=generate_registration_id(), + adv_secret_b64="AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + registered=True, + jid="111:7@s.whatsapp.net", + ) + adapter._save(creds) + wire = GroupWire() + adapter.session = wire # type: ignore[assignment] + adapter._connected = True + + async def fake_sessions(self, _wire, _creds, addresses): + return Sessions(addresses) + adapter._ensure_signal_sessions = types.MethodType(fake_sessions, adapter) # type: ignore[method-assign] + + first = await adapter.send_text(jid="12345@g.us", text="hello group") + self.assertTrue(first["server_ack"]) + self.assertEqual(first["sender_key_recipients"], 2) + self.assertEqual(first["device_fanout"], 2) + stanza = wire.sent[-1] + self.assertEqual(stanza.attrs["to"], "12345@g.us") + self.assertEqual(stanza.attrs["addressing_mode"], "pn") + self.assertEqual(stanza.child("enc").attrs["type"], "skmsg") + self.assertEqual(len(stanza.child("participants").children("to")), 2) + + second = await adapter.send_text(jid="12345@g.us", text="second") + self.assertTrue(second["server_ack"]) + self.assertEqual(second["sender_key_recipients"], 0) + stanza2 = wire.sent[-1] + self.assertIsNone(stanza2.child("participants")) + memory = auth.get_json("group-sender-key-memory", "12345@g.us") + self.assertEqual( + set(memory["devices"]), + {"111:1@s.whatsapp.net", "222@s.whatsapp.net"}, + ) + self.assertTrue(adapter.status()["group_text_send_implemented"]) + self.assertFalse(adapter.status()["groups_ready"]) + + async def test_group_media_send_uses_sender_key_and_enc_mediatype(self): + metadata_response = BinaryNode( + "iq", + {"type": "result"}, + [ + BinaryNode( + "group", + {"id": "12345", "subject": "Test", "addressing_mode": "pn"}, + [ + BinaryNode("participant", {"jid": "111@s.whatsapp.net"}), + BinaryNode("participant", {"jid": "222@s.whatsapp.net"}), + ], + ) + ], + ) + usync_response = BinaryNode( + "iq", + {"type": "result"}, + [ + BinaryNode( + "usync", + {}, + [ + BinaryNode( + "list", + {}, + [ + BinaryNode( + "user", + {"jid": "111@s.whatsapp.net"}, + [ + BinaryNode( + "devices", + {}, + [ + BinaryNode( + "device-list", + {}, + [BinaryNode("device", {"id": "1", "key-index": "7"})], + ) + ], + ) + ], + ), + BinaryNode( + "user", + {"jid": "222@s.whatsapp.net"}, + [ + BinaryNode( + "devices", + {}, + [ + BinaryNode( + "device-list", + {}, + [BinaryNode("device", {"id": "0"})], + ) + ], + ) + ], + ), + ], + ) + ], + ) + ], + ) + media_conn_response = BinaryNode( + "iq", + {"type": "result"}, + [ + BinaryNode( + "media_conn", + {"auth": "auth-token", "ttl": "60"}, + [BinaryNode("host", {"hostname": "mmg.example.com"})], + ) + ], + ) + + class Signal: + def encrypt(self, data): + return "msg", b"pairwise-" + bytes(data[:4]) + + class Sessions: + def __init__(self, jids): + self.items = {jid: Signal() for jid in jids} + def load(self, jid): + return self.items.get(jid) + def save(self, jid, signal): + self.items[jid] = signal + + class FileStore: + def raw_bytes(self, file_id): + self.last = file_id + return ( + {"filename": "report.pdf", "media_type": "application/pdf"}, + b"stored-file-payload", + ) + + class Wire: + def __init__(self): + self.closed = False + self.sent = [] + async def query(self, node, *, timeout=30.0): + if node.attrs.get("xmlns") == "w:m": + return media_conn_response + if node.attrs.get("xmlns") == "w:g2": + return metadata_response + if node.attrs.get("xmlns") == "usync": + return usync_response + raise AssertionError(f"unexpected query {node.attrs}") + async def send_and_wait(self, node, *, response_tag=None, timeout=30.0): + self.sent.append(node) + return BinaryNode("ack", {"id": node.attrs["id"], "class": "message"}) + + upload = MediaUpload( + media_type="document", + url="https://mmg.example.com/file", + direct_path="/v/t62/file", + media_key=b"k" * 32, + file_sha256=b"s" * 32, + file_enc_sha256=b"e" * 32, + file_length=len(b"stored-file-payload"), + mimetype="application/pdf", + filename="report.pdf", + caption="Quarterly", + media_key_timestamp=1700000000, + ) + + with TemporaryDirectory() as td: + auth = EncryptedAuthStore(str(Path(td) / "auth.db")) + store = FileStore() + adapter = CurrentProtocolAdapter(auth, file_store=store) + identity = generate_curve_keypair() + creds = ProtocolCredentials( + noise=generate_curve_keypair(), + identity=identity, + signed_pre_key=generate_signed_pre_key(identity, 1), + registration_id=generate_registration_id(), + adv_secret_b64="AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + registered=True, + jid="111:7@s.whatsapp.net", + ) + adapter._save(creds) + wire = Wire() + adapter.session = wire # type: ignore[assignment] + adapter._connected = True + + async def fake_sessions(self, _wire, _creds, addresses): + return Sessions(addresses) + adapter._ensure_signal_sessions = types.MethodType(fake_sessions, adapter) # type: ignore[method-assign] + + with patch( + "postmaster.whatsapp_v990.adapter.upload_media_bytes", + new=AsyncMock(return_value=(upload, b"encrypted-media")), + ): + result = await adapter.send_media( + jid="12345@g.us", + stored_file_id="stored-1", + caption="Quarterly", + ) + + self.assertTrue(result["server_ack"]) + self.assertTrue(result["group"]) + self.assertEqual(result["media_type"], "document") + self.assertEqual(result["stored_file_id"], "stored-1") + self.assertEqual(store.last, "stored-1") + stanza = wire.sent[-1] + self.assertNotIn("mediatype", stanza.attrs) + self.assertEqual(stanza.child("enc").attrs["type"], "skmsg") + self.assertEqual(stanza.child("enc").attrs["mediatype"], "document") + self.assertTrue(adapter.status()["group_media_send_implemented"]) + self.assertFalse(adapter.status()["groups_ready"]) + + async def test_group_media_receive_downloads_and_saves_canonical_stored_file(self): + class ReceiveStore: + def __init__(self): + self.saved = [] + def save_bytes(self, **kwargs): + self.saved.append(kwargs) + return {"id": "received-media-1", "filename": kwargs["filename"]} + + with TemporaryDirectory() as td: + sender_auth = EncryptedAuthStore(str(Path(td) / "sender.db")) + receiver_auth = EncryptedAuthStore(str(Path(td) / "receiver.db")) + author = "222:1@s.whatsapp.net" + group = "12345@g.us" + upload = MediaUpload( + media_type="document", + url="https://mmg.example.com/file", + direct_path="/v/t62/file", + media_key=b"k" * 32, + file_sha256=b"s" * 32, + file_enc_sha256=b"e" * 32, + file_length=17, + mimetype="application/pdf", + filename="report.pdf", + caption="Inbound report", + media_key_timestamp=1700000000, + ) + media_proto = encode_media_message(upload) + + sender_keys = EncryptedSenderKeyStore(sender_auth) + distribution = sender_keys.distribution(group, author) + ciphertext = sender_keys.encrypt( + group, + author, + pad_random_max16(media_proto, random1=b"\x00"), + ) + distribution_message = encode_sender_key_distribution_message(group, distribution) + stored = ReceiveStore() + captured = [] + adapter = CurrentProtocolAdapter( + receiver_auth, + file_store=stored, + file_owner_id="davide", + file_project_id="postmaster-mcp", + on_message=lambda **kwargs: captured.append(kwargs), + ) + identity = generate_curve_keypair() + creds = ProtocolCredentials( + noise=generate_curve_keypair(), + identity=identity, + signed_pre_key=generate_signed_pre_key(identity, 1), + registration_id=generate_registration_id(), + adv_secret_b64="AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + registered=True, + jid="111:7@s.whatsapp.net", + ) + adapter._save(creds) + + async def fake_pairwise(self, node, loaded_creds, *, e2e_type, ciphertext): + return distribution_message + adapter._decrypt_pairwise_message_proto = types.MethodType(fake_pairwise, adapter) # type: ignore[method-assign] + + stanza = BinaryNode( + "message", + {"from": group, "participant": author, "id": "group-media-1"}, + [ + BinaryNode("enc", {"v": "2", "type": "msg"}, b"pairwise-skdm"), + BinaryNode("enc", {"v": "2", "type": "skmsg"}, ciphertext), + ], + ) + with patch( + "postmaster.whatsapp_v990.adapter.download_media", + new=AsyncMock(return_value=b"downloaded-payload"), + ): + await adapter._handle_incoming_message(stanza) + + self.assertEqual(len(captured), 1) + self.assertEqual(captured[0]["jid"], group) + self.assertEqual(captured[0]["kind"], "media") + self.assertEqual(captured[0]["text"], "Inbound report") + self.assertEqual(captured[0]["stored_file_id"], "received-media-1") + self.assertEqual(stored.saved[0]["owner_id"], "davide") + self.assertEqual(stored.saved[0]["project_id"], "postmaster-mcp") + self.assertEqual(stored.saved[0]["filename"], "report.pdf") + self.assertEqual(stored.saved[0]["data"], b"downloaded-payload") + self.assertIn("whatsapp", stored.saved[0]["tags"]) + self.assertTrue(adapter.status()["media_receive_to_stored_file_implemented"]) + + async def test_group_receive_processes_pairwise_distribution_before_skmsg(self): + with TemporaryDirectory() as td: + sender_auth = EncryptedAuthStore(str(Path(td) / "sender.db")) + receiver_auth = EncryptedAuthStore(str(Path(td) / "receiver.db")) + author = "222:1@s.whatsapp.net" + group = "12345@g.us" + + sender_keys = EncryptedSenderKeyStore(sender_auth) + distribution = sender_keys.distribution(group, author) + ciphertext = sender_keys.encrypt( + group, + author, + pad_random_max16(encode_text_message("hello inbound group"), random1=b"\x00"), + ) + distribution_message = encode_sender_key_distribution_message(group, distribution) + + captured = [] + adapter = CurrentProtocolAdapter( + receiver_auth, + on_message=lambda **kwargs: captured.append(kwargs), + ) + identity = generate_curve_keypair() + creds = ProtocolCredentials( + noise=generate_curve_keypair(), + identity=identity, + signed_pre_key=generate_signed_pre_key(identity, 1), + registration_id=generate_registration_id(), + adv_secret_b64="AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + registered=True, + jid="111:7@s.whatsapp.net", + ) + adapter._save(creds) + + async def fake_pairwise(self, node, loaded_creds, *, e2e_type, ciphertext): + self.assert_creds = loaded_creds + return distribution_message + adapter._decrypt_pairwise_message_proto = types.MethodType(fake_pairwise, adapter) # type: ignore[method-assign] + + stanza = BinaryNode( + "message", + {"from": group, "participant": author, "id": "group-msg-1"}, + [ + BinaryNode("enc", {"v": "2", "type": "msg"}, b"pairwise-skdm"), + BinaryNode("enc", {"v": "2", "type": "skmsg"}, ciphertext), + ], + ) + await adapter._handle_incoming_message(stanza) + + self.assertEqual(len(captured), 1) + self.assertEqual(captured[0]["jid"], group) + self.assertEqual(captured[0]["direction"], "in") + self.assertEqual(captured[0]["kind"], "text") + self.assertEqual(captured[0]["text"], "hello inbound group") + receiver_keys = EncryptedSenderKeyStore(receiver_auth) + self.assertIsNotNone(receiver_keys.load(group, author)) + self.assertTrue(adapter.status()["group_text_receive_implemented"]) + self.assertFalse(adapter.status()["groups_ready"]) + + async def test_adapter_list_groups_uses_live_session_query(self): + response = BinaryNode("iq", {"type": "result"}, [BinaryNode("groups", {}, [group_node()])]) + with TemporaryDirectory() as td: + auth = EncryptedAuthStore(str(Path(td) / "auth.db")) + adapter = CurrentProtocolAdapter(auth) + identity = generate_curve_keypair() + creds = ProtocolCredentials( + noise=generate_curve_keypair(), + identity=identity, + signed_pre_key=generate_signed_pre_key(identity, 1), + registration_id=generate_registration_id(), + adv_secret_b64="AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + registered=True, + jid="111:7@s.whatsapp.net", + ) + adapter._save(creds) + wire = FakeWire(response) + adapter.session = wire # type: ignore[assignment] + adapter._connected = True + groups = await adapter.list_groups() + self.assertEqual(groups[0]["id"], "12345@g.us") + self.assertEqual(wire.queries[0].attrs["xmlns"], "w:g2") + self.assertTrue(adapter.status()["group_listing_implemented"]) + self.assertFalse(adapter.status()["groups_ready"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_whatsapp_media_v990.py b/tests/test_whatsapp_media_v990.py new file mode 100644 index 00000000..16ac451f --- /dev/null +++ b/tests/test_whatsapp_media_v990.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +import base64 +import json +import unittest + +import httpx + +from postmaster.whatsapp_v990.binary import BinaryNode +from postmaster.whatsapp_v990.media import ( + MediaConn, + MediaHost, + MediaUpload, + build_media_conn_query, + decode_media_descriptor, + download_media, + encode_media_message, + infer_media_type, + parse_media_conn, + upload_media_bytes, + upload_token, +) + + +class WhatsAppMediaV990Tests(unittest.IsolatedAsyncioTestCase): + def test_media_conn_query_and_parse(self): + query=build_media_conn_query() + self.assertEqual(query.tag,"iq") + self.assertEqual(query.attrs["xmlns"],"w:m") + self.assertEqual(query.child("media_conn").tag,"media_conn") + result=BinaryNode("iq",{"type":"result"},[ + BinaryNode("media_conn",{"auth":"token secret","ttl":"3600"},[ + BinaryNode("host",{"hostname":"mmg.example.test","maxContentLengthBytes":"1000000"}) + ]) + ]) + conn=parse_media_conn(result) + self.assertEqual(conn.auth,"token secret") + self.assertEqual(conn.ttl,3600) + self.assertEqual(conn.hosts[0].hostname,"mmg.example.test") + self.assertEqual(conn.hosts[0].max_content_length_bytes,1000000) + + def test_infer_type_and_upload_token(self): + self.assertEqual(infer_media_type(mimetype="image/jpeg",filename="x.jpg"),"image") + self.assertEqual(infer_media_type(mimetype="image/webp",filename="x.webp"),"sticker") + self.assertEqual(infer_media_type(mimetype="application/pdf",filename="x.pdf"),"document") + token=upload_token(bytes(range(32))) + self.assertNotIn("=",token) + self.assertNotIn("+",token) + self.assertNotIn("/",token) + + async def test_upload_encrypts_posts_and_returns_metadata(self): + seen={} + async def handler(request: httpx.Request): + seen["url"]=str(request.url) + seen["origin"]=request.headers.get("origin") + seen["content_type"]=request.headers.get("content-type") + seen["body"]=await request.aread() + return httpx.Response(200,json={ + "url":"https://mmg.example.test/v/t62.7118/file", + "direct_path":"/v/t62.7118/file", + }) + client=httpx.AsyncClient(transport=httpx.MockTransport(handler)) + try: + upload,encrypted=await upload_media_bytes( + b"hello stored file", + media_type="document", + mimetype="text/plain", + filename="note.txt", + caption="caption", + media_conn=MediaConn("auth token",3600,(MediaHost("mmg.example.test",100000),)), + client=client, + ) + finally: + await client.aclose() + self.assertEqual(upload.media_type,"document") + self.assertEqual(upload.filename,"note.txt") + self.assertEqual(upload.caption,"caption") + self.assertEqual(upload.file_length,len(b"hello stored file")) + self.assertEqual(seen["origin"],"https://web.whatsapp.com") + self.assertEqual(seen["content_type"],"application/octet-stream") + self.assertEqual(seen["body"],encrypted) + self.assertIn("/mms/document/",seen["url"]) + self.assertIn("auth=auth%20token",seen["url"]) + + async def test_media_proto_roundtrip_for_supported_types(self): + for kind,mimetype,filename,caption in ( + ("image","image/jpeg",None,"image cap"), + ("document","application/pdf","file.pdf","document cap"), + ("audio","audio/ogg",None,""), + ("video","video/mp4",None,"video cap"), + ("sticker","image/webp","sticker.webp",""), + ): + upload=MediaUpload( + media_type=kind, + url="https://mmg.example.test/file", + direct_path="/file", + media_key=b"k"*32, + file_sha256=b"s"*32, + file_enc_sha256=b"e"*32, + file_length=1234, + mimetype=mimetype, + filename=filename, + caption=caption, + media_key_timestamp=1790000000, + ) + raw=encode_media_message(upload) + decoded=decode_media_descriptor(raw) + self.assertIsNotNone(decoded) + self.assertEqual(decoded.media_type,kind) + self.assertEqual(decoded.url,upload.url) + self.assertEqual(decoded.direct_path,upload.direct_path) + self.assertEqual(decoded.media_key,upload.media_key) + self.assertEqual(decoded.file_sha256,upload.file_sha256) + self.assertEqual(decoded.file_enc_sha256,upload.file_enc_sha256) + self.assertEqual(decoded.file_length,upload.file_length) + self.assertEqual(decoded.mimetype,mimetype) + if kind=="document": + self.assertEqual(decoded.filename,filename) + self.assertEqual(decoded.caption,caption) + elif kind in {"image","video"}: + self.assertEqual(decoded.caption,caption) + + async def test_download_verifies_encrypted_hash_mac_plain_hash_and_length(self): + plaintext=b"media payload"*100 + async def upload_handler(request: httpx.Request): + body=await request.aread() + upload_handler.encrypted=bytes(body) + return httpx.Response(200,json={"direct_path":"/media/abc"}) + upload_handler.encrypted=b"" + upload_client=httpx.AsyncClient(transport=httpx.MockTransport(upload_handler)) + try: + upload,_=await upload_media_bytes( + plaintext, + media_type="image", + mimetype="image/jpeg", + filename="x.jpg", + media_conn=MediaConn("a",3600,(MediaHost("mmg.example.test",100000),)), + client=upload_client, + ) + finally: + await upload_client.aclose() + descriptor=decode_media_descriptor(encode_media_message(upload)) + async def download_handler(request: httpx.Request): + self.assertEqual(request.headers.get("origin"),"https://web.whatsapp.com") + return httpx.Response(200,content=upload_handler.encrypted) + download_client=httpx.AsyncClient(transport=httpx.MockTransport(download_handler)) + try: + restored=await download_media(descriptor,client=download_client,default_host="mmg.example.test") + finally: + await download_client.aclose() + self.assertEqual(restored,plaintext) + + +if __name__=="__main__": + unittest.main() diff --git a/tests/test_whatsapp_messages_v990.py b/tests/test_whatsapp_messages_v990.py new file mode 100644 index 00000000..ce07799d --- /dev/null +++ b/tests/test_whatsapp_messages_v990.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import unittest + +from postmaster.whatsapp_v990.binary import BinaryNode +from postmaster.whatsapp_v990.messages import ( + build_ack_stanza, + build_direct_message_stanza, + decode_text_message, + encode_device_sent_message, + encode_reply_text_message, + encode_text_message, + encrypted_participant_node, + generate_message_id_v2, + pad_random_max16, + participant_hash_v2, + unpad_random_max16, +) +from postmaster.whatsapp_v990.proto import decode_fields + + +class WhatsAppMessagesV990Tests(unittest.TestCase): + def test_message_id_and_participant_hash_are_deterministic(self): + mid=generate_message_id_v2( + "123@s.whatsapp.net", + now_seconds=1720000000, + random16=bytes(range(16)), + ) + self.assertEqual(mid,"3EB0"+mid[4:]) + self.assertEqual(len(mid),22) + self.assertEqual(mid,generate_message_id_v2("123@s.whatsapp.net",now_seconds=1720000000,random16=bytes(range(16)))) + self.assertEqual( + participant_hash_v2(["2@s.whatsapp.net","1@s.whatsapp.net"]), + participant_hash_v2(["1@s.whatsapp.net","2@s.whatsapp.net"]), + ) + self.assertTrue(participant_hash_v2(["1@s.whatsapp.net"]).startswith("2:")) + + def test_whatsapp_message_padding_roundtrip_and_bounds(self): + raw=b"hello protobuf" + for low_nibble in (0,1,7,15): + padded=pad_random_max16(raw,random1=bytes((low_nibble,))) + self.assertEqual(len(padded)-len(raw),low_nibble+1) + self.assertEqual(unpad_random_max16(padded),raw) + with self.assertRaises(Exception): + unpad_random_max16(b"") + with self.assertRaises(Exception): + unpad_random_max16(b"x"+bytes((17,))) + + def test_text_and_device_sent_message_wire_fields(self): + msg=encode_text_message("hello") + fields=decode_fields(msg) + self.assertEqual(fields[0].number,1) + self.assertEqual(fields[0].value,b"hello") + + dsm=encode_device_sent_message("123@s.whatsapp.net",msg,phash="2:ABCDEF") + top=decode_fields(dsm) + self.assertEqual(top[0].number,31) + inner=decode_fields(top[0].value) + self.assertEqual(inner[0].number,1) + self.assertEqual(inner[0].value,b"123@s.whatsapp.net") + self.assertEqual(inner[1].number,2) + self.assertEqual(inner[1].value,msg) + self.assertEqual(inner[2].number,3) + self.assertEqual(inner[2].value,b"2:ABCDEF") + + def test_text_decoder_handles_conversation_extended_and_device_sent(self): + plain=encode_text_message("hello") + self.assertEqual(decode_text_message(plain),"hello") + extended=encode_reply_text_message( + "reply",stanza_id="m1",participant="123@s.whatsapp.net",remote_jid="123@s.whatsapp.net" + ) + self.assertEqual(decode_text_message(extended),"reply") + dsm=encode_device_sent_message("123@s.whatsapp.net",plain,phash="2:ABCDEF") + self.assertEqual(decode_text_message(dsm),"hello") + + def test_transport_ack_is_not_read_receipt(self): + incoming=BinaryNode( + "message", + {"id":"m1","from":"123:2@lid","type":"text","participant":"123:2@lid"}, + ) + ack=build_ack_stanza(incoming,me_id="999:4@s.whatsapp.net") + self.assertEqual(ack.tag,"ack") + self.assertEqual(ack.attrs["id"],"m1") + self.assertEqual(ack.attrs["to"],"123:2@lid") + self.assertEqual(ack.attrs["class"],"message") + self.assertEqual(ack.attrs["from"],"999:4@s.whatsapp.net") + self.assertEqual(ack.attrs["type"],"text") + self.assertNotEqual(ack.attrs.get("type"),"read") + + def test_direct_stanza_participants_and_device_identity(self): + p1=encrypted_participant_node("123@s.whatsapp.net",ciphertext_type="pkmsg",ciphertext=b"a") + p2=encrypted_participant_node("123:2@s.whatsapp.net",ciphertext_type="msg",ciphertext=b"b") + stanza=build_direct_message_stanza( + destination_jid="123@s.whatsapp.net", + message_id="3EB0TEST", + participants=[p1,p2], + device_identity=b"adv", + ) + self.assertEqual(stanza.tag,"message") + self.assertEqual(stanza.attrs["to"],"123@s.whatsapp.net") + self.assertEqual(stanza.attrs["type"],"text") + self.assertTrue(stanza.attrs["phash"].startswith("2:")) + self.assertEqual(len(stanza.child("participants").children("to")),2) + self.assertEqual(stanza.child("device-identity").content,b"adv") + + +if __name__=="__main__": + unittest.main() diff --git a/tests/test_whatsapp_protocol_v990.py b/tests/test_whatsapp_protocol_v990.py new file mode 100644 index 00000000..56470019 --- /dev/null +++ b/tests/test_whatsapp_protocol_v990.py @@ -0,0 +1,295 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path +import sqlite3 +import stat +from tempfile import TemporaryDirectory +import unittest + +from postmaster.whatsapp_v990.binary import BinaryNode, BinaryNodeCodec, BinaryNodeError, TokenTable +from postmaster.whatsapp_v990.jid import JID, parse_jid, same_user, transfer_device +from postmaster.whatsapp_v990.proto import decode_fields, field_bytes, field_message, field_varint +from postmaster.whatsapp_v990.receipts import receipt_for_event +from postmaster.whatsapp_v990.store import EncryptedAuthStore +from postmaster.whatsapp_v990.transport import ReconnectingTransport, ReconnectPolicy, TransportError + + +class JIDAndBinaryTests(unittest.TestCase): + def test_jid_device_agent_and_normalization(self): + jid = parse_jid("12345_7:42@c.us") + self.assertEqual(jid.user, "12345") + self.assertEqual(jid.agent, 7) + self.assertEqual(jid.device, 42) + self.assertEqual(str(jid), "12345_7:42@c.us") + self.assertEqual(str(jid.normalized_user()), "12345@s.whatsapp.net") + self.assertTrue(same_user("12345:1@s.whatsapp.net", "12345:9@s.whatsapp.net")) + self.assertEqual(str(transfer_device("12345:9@s.whatsapp.net", "999@g.us")), "999:9@g.us") + + def test_binary_node_roundtrip_raw_jid_bytes_children_and_compression(self): + codec = BinaryNodeCodec() + node = BinaryNode( + "iq", + {"id": "123.45", "to": "12345:7@s.whatsapp.net", "type": "get"}, + [BinaryNode("query", {"xmlns": "w:g2"}), BinaryNode("payload", {}, b"\x00\x01hello")], + ) + for compressed in (False, True): + encoded = codec.encode(node, compressed=compressed) + decoded = codec.decode(encoded) + self.assertEqual(decoded.tag, "iq") + self.assertEqual(decoded.attrs["to"], "12345:7@s.whatsapp.net") + self.assertEqual(decoded.children()[0].attrs["xmlns"], "w:g2") + self.assertEqual(decoded.children()[1].content, b"\x00\x01hello") + + def test_binary_codec_uses_injected_token_table_and_rejects_unknown(self): + tokens = TokenTable([None, "iq", "type", "get"], [["query"]], version=3) + codec = BinaryNodeCodec(tokens) + encoded = codec.encode(BinaryNode("iq", {"type": "get"}, [BinaryNode("query")])) + self.assertEqual(codec.decode(encoded).children()[0].tag, "query") + broken = bytearray(encoded) + # Root list prefix is 0, LIST_8, size, then tag token. Replace tag token with unknown 200. + broken[3] = 200 + with self.assertRaises(BinaryNodeError): + codec.decode(bytes(broken)) + + +class CurrentTokenTableTests(unittest.TestCase): + def test_pair_device_tokens_are_pinned_to_current_snapshot(self): + self.assertEqual(CURRENT_TOKEN_TABLE.version, 20260921) + self.assertEqual(CURRENT_TOKEN_TABLE.token_for("iq"), (None, 25)) + self.assertEqual(CURRENT_TOKEN_TABLE.token_for("pair-device"), (1, 238)) + self.assertEqual(CURRENT_TOKEN_TABLE.token_for("ref"), (1, 80)) + + def test_fb_and_interop_jid_wire_tags_decode(self): + codec = BinaryNodeCodec(CURRENT_TOKEN_TABLE) + # FB_JID: user string, 16-bit device, server string. + fb = bytes([246, 252, 3]) + b"123" + bytes([0, 7, 3]) + value, pos = codec._read_string(fb, 1, first=246) + self.assertEqual(value, "123:7@s.whatsapp.net") + self.assertEqual(pos, len(fb)) + + # INTEROP_JID: user string, 16-bit device, 16-bit integrator, optional server. + interop = bytes([245, 252, 3]) + b"456" + bytes([0, 2, 0, 9, 3]) + value, pos = codec._read_string(interop, 1, first=245) + self.assertEqual(value, "9-456:2@s.whatsapp.net") + self.assertEqual(pos, len(interop)) + + +class ProtoTests(unittest.TestCase): + def test_minimal_wire_codec_roundtrip(self): + nested = field_varint(1, 7) + field_bytes(2, "desktop") + raw = field_varint(1, 300) + field_bytes(2, b"abc") + field_message(3, [nested]) + fields = decode_fields(raw) + self.assertEqual(fields[0].value, 300) + self.assertEqual(fields[1].value, b"abc") + nested_fields = decode_fields(fields[2].value) + self.assertEqual([x.number for x in nested_fields], [1, 2]) + + +class StoreAndReceiptTests(unittest.TestCase): + def test_encrypted_store_persists_without_plaintext_or_key_exposure(self): + with TemporaryDirectory() as td: + db = Path(td) / "wa.db" + store = EncryptedAuthStore(str(db)) + secret = {"jid":"123@s.whatsapp.net", "private":"super-secret-private-material"} + store.put_json("creds", "primary", secret) + self.assertEqual(store.get_json("creds", "primary"), secret) + raw_db = db.read_bytes() + self.assertNotIn(b"super-secret-private-material", raw_db) + self.assertEqual(stat.S_IMODE((Path(str(db)+".key")).stat().st_mode) & 0o077, 0) + status = store.status() + self.assertTrue(status["encrypted_at_rest"]) + self.assertFalse(status["private_material_exposed"]) + + def test_receipts_are_asymmetric(self): + self.assertFalse(receipt_for_event(event="read").emit) + self.assertFalse(receipt_for_event(event="send", outbound_action=True).emit) + reply = receipt_for_event(event="reply", outbound_action=True, reply_to_message=True) + self.assertTrue(reply.emit) + self.assertEqual(reply.receipt_type, "read") + + +class FakeSocket: + def __init__(self): self.sent=[]; self.incoming=[b"ok"]; self.closed=False + async def send(self, data): self.sent.append(bytes(data)) + async def recv(self): return self.incoming.pop(0) + async def close(self): self.closed=True + + +class TransportTests(unittest.IsolatedAsyncioTestCase): + async def test_reconnect_then_send_receive_close(self): + calls = 0 + sock = FakeSocket() + async def connector(): + nonlocal calls + calls += 1 + if calls == 1: raise OSError("temporary") + return sock + sleeps=[] + async def no_sleep(delay): sleeps.append(delay) + transport = ReconnectingTransport(connector, policy=ReconnectPolicy(base_seconds=0.1, jitter=0)) + await transport.ensure_connected(max_attempts=2, sleep=no_sleep) + self.assertTrue(transport.connected) + self.assertEqual(transport.reconnects, 1) + self.assertEqual(sleeps, [0.1]) + await transport.send(b"hello") + self.assertEqual(sock.sent, [b"hello"]) + self.assertEqual(await transport.recv(), b"ok") + await transport.close() + self.assertTrue(sock.closed) + self.assertFalse(transport.connected) + + async def test_send_requires_connection(self): + async def connector(): return FakeSocket() + transport = ReconnectingTransport(connector) + with self.assertRaises(TransportError): + await transport.send(b"x") + + +if __name__ == "__main__": + unittest.main() + +from postmaster.whatsapp_v990.handshake import ( + HandshakeError, HandshakeMessage, ServerHello, decode_handshake, encode_client_finish, encode_client_hello, encode_handshake, +) +from postmaster.whatsapp_v990.pairing import PAIRING_QR_PREFIX, PairingQR, build_pairing_qr_data, companion_web_client_type +from postmaster.whatsapp_v990.websocket_driver import WebSocketDriverConfig, WebSocketDriverError, open_whatsapp_websocket +from postmaster.whatsapp_v990.tokens import CURRENT_TOKEN_TABLE + + +class HandshakeAndPairingTests(unittest.IsolatedAsyncioTestCase): + def test_handshake_client_hello_and_server_hello_wire_fields(self): + eph = bytes(range(32)) + raw = encode_client_hello(eph) + top = decode_fields(raw) + self.assertEqual(top[0].number, 2) + decoded = decode_handshake(raw) + self.assertEqual(decoded.client_hello.ephemeral, eph) + + server = encode_handshake(HandshakeMessage(server_hello=ServerHello(eph, b"static", b"payload"))) + parsed = decode_handshake(server) + self.assertEqual(parsed.server_hello.static, b"static") + self.assertEqual(parsed.server_hello.payload, b"payload") + + def test_handshake_client_finish_and_validation(self): + raw = encode_client_finish(b"encrypted-static", b"encrypted-payload") + parsed = decode_handshake(raw) + self.assertEqual(parsed.client_finish.static, b"encrypted-static") + self.assertEqual(parsed.client_finish.payload, b"encrypted-payload") + with self.assertRaises(HandshakeError): + encode_client_hello(b"too-short") + + def test_current_companion_qr_shape_and_public_status(self): + payload = build_pairing_qr_data( + "ref-token", b"n" * 32, b"i" * 32, "adv-secret-b64", os_name="Linux", browser_name="Chrome" + ) + self.assertTrue(payload.startswith(PAIRING_QR_PREFIX)) + fragment = payload.split("#", 1)[1] + self.assertEqual(len(fragment.split(",")), 5) + self.assertEqual(fragment.split(",")[-1], "1") + self.assertEqual(companion_web_client_type("Windows", "Desktop"), 8) + self.assertEqual(companion_web_client_type("Linux", "Desktop"), 7) + status = PairingQR(payload).public_status() + self.assertTrue(status["available"]) + self.assertFalse(status["payload_exposed_in_status"]) + self.assertNotIn("payload", status) + + async def test_websocket_driver_uses_current_endpoint_origin_and_can_be_injected(self): + calls = [] + class Sock: pass + async def connector(*args, **kwargs): + calls.append((args, kwargs)); return Sock() + sock = await open_whatsapp_websocket(connect_impl=connector) + self.assertIsInstance(sock, Sock) + self.assertEqual(calls[0][0][0], "wss://web.whatsapp.com/ws/chat") + self.assertEqual(calls[0][1]["origin"], "https://web.whatsapp.com") + with self.assertRaises(WebSocketDriverError): + await open_whatsapp_websocket(WebSocketDriverConfig(url="ws://insecure.invalid"), connect_impl=connector) + +from postmaster.whatsapp_v990.crypto import generate_curve_keypair, signal_public_key +from postmaster.whatsapp_v990.signal_keys import ( + chain_message_seed, derive_message_keys, derive_x3dh_initiator, derive_x3dh_responder, + generate_registration_id, generate_signed_pre_key, next_chain_key, verify_signed_pre_key, +) +from postmaster.whatsapp_v990.signal_wire import PreKeyWhisperMessageV3, SignalMessageV3, SignalWireError + + +class SignalV3WireAndKeyTests(unittest.TestCase): + def test_signed_prekey_matches_whatsapp_bundle_shape(self): + identity = generate_curve_keypair() + signed_pair = generate_curve_keypair() + signed = generate_signed_pre_key(identity, 1, pre_key=signed_pair, random64=b"r" * 64) + self.assertEqual(signed.key_id, 1) + self.assertTrue(verify_signed_pre_key(identity.public, signed.key_pair.public, signed.signature)) + self.assertFalse(verify_signed_pre_key(generate_curve_keypair().public, signed.key_pair.public, signed.signature)) + self.assertEqual(generate_registration_id(b"\xff\xff"), 0x3FFF) + + def test_x3dh_initiator_responder_match_with_and_without_one_time_prekey(self): + alice_identity = generate_curve_keypair() + alice_base = generate_curve_keypair() + bob_identity = generate_curve_keypair() + bob_signed = generate_curve_keypair() + bob_one = generate_curve_keypair() + for one_time in (None, bob_one): + a = derive_x3dh_initiator( + our_identity_private=alice_identity.private, + our_base_private=alice_base.private, + their_identity_public=bob_identity.public, + their_signed_pre_key_public=bob_signed.public, + their_one_time_pre_key_public=None if one_time is None else one_time.public, + ) + b = derive_x3dh_responder( + our_identity_private=bob_identity.private, + our_signed_pre_key_private=bob_signed.private, + their_identity_public=alice_identity.public, + their_base_public=alice_base.public, + our_one_time_pre_key_private=None if one_time is None else one_time.private, + ) + self.assertEqual(a, b) + self.assertEqual(len(a[0]), 32) + self.assertEqual(len(a[1]), 32) + + def test_signal_chain_key_reference_vector(self): + chain_key = bytes.fromhex("8ab72d6f4cc5ac0d387eaf463378ddb28edd07385b1cb01250c715982e7ad48f") + cipher_key, mac_key, _iv = derive_message_keys(chain_key) + self.assertEqual(cipher_key.hex(), "bf51e9d75e0e31031051f82a2491ffc084fa298b7793bd9db620056febf45217") + self.assertEqual(mac_key.hex(), "c6c77d6a73a354337a56435e34607dfe48e3ace14e77314dc6abc172e7a7030b") + self.assertEqual(next_chain_key(chain_key).hex(), "28e8f8fee54b801eef7c5cfb2f17f32c7b334485bbb70fac6ec10342a246d15d") + self.assertNotEqual(chain_message_seed(chain_key), next_chain_key(chain_key)) + + def test_signal_v3_message_wire_and_mac(self): + ratchet = generate_curve_keypair() + sender_id = generate_curve_keypair() + receiver_id = generate_curve_keypair() + mac_key = b"m" * 32 + message = SignalMessageV3(ratchet.public, 7, 3, b"ciphertext") + raw = message.serialize(mac_key=mac_key, sender_identity_key=sender_id.public, receiver_identity_key=receiver_id.public) + self.assertEqual(raw[0], 0x33) + parsed = SignalMessageV3.parse(raw) + self.assertEqual(parsed.ratchet_key, signal_public_key(ratchet.public)) + self.assertEqual(parsed.counter, 7) + self.assertEqual(parsed.previous_counter, 3) + self.assertTrue(parsed.verify_mac(mac_key=mac_key, sender_identity_key=sender_id.public, receiver_identity_key=receiver_id.public)) + tampered = raw[:-1] + bytes([raw[-1] ^ 1]) + self.assertFalse(SignalMessageV3.parse(tampered).verify_mac(mac_key=mac_key, sender_identity_key=sender_id.public, receiver_identity_key=receiver_id.public)) + + def test_prekey_whisper_v3_roundtrip_and_version_rejection(self): + base = generate_curve_keypair() + identity = generate_curve_keypair() + nested = b"\x33nested-signal-message" + msg = PreKeyWhisperMessageV3( + registration_id=1234, + pre_key_id=9, + signed_pre_key_id=1, + base_key=base.public, + identity_key=identity.public, + message=nested, + ) + raw = msg.serialize() + parsed = PreKeyWhisperMessageV3.parse(raw) + self.assertEqual(parsed.registration_id, 1234) + self.assertEqual(parsed.pre_key_id, 9) + self.assertEqual(parsed.base_key, signal_public_key(base.public)) + self.assertEqual(parsed.message, nested) + with self.assertRaises(SignalWireError): + PreKeyWhisperMessageV3.parse(bytes([0x44]) + raw[1:]) diff --git a/tests/test_whatsapp_runtime_v990.py b/tests/test_whatsapp_runtime_v990.py new file mode 100644 index 00000000..86b1a602 --- /dev/null +++ b/tests/test_whatsapp_runtime_v990.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +from pathlib import Path +import sys +import types +from tempfile import TemporaryDirectory +from types import SimpleNamespace +import unittest +from starlette.applications import Starlette + +mcp_mod = sys.modules.setdefault("mcp", types.ModuleType("mcp")) +mcp_types = sys.modules.setdefault("mcp.types", types.ModuleType("mcp.types")) +if not hasattr(mcp_types, "ToolAnnotations"): + class ToolAnnotations: + def __init__(self, **kwargs): self.__dict__.update(kwargs) + mcp_types.ToolAnnotations = ToolAnnotations + +from postmaster.whatsapp_runtime_v990 import MCP_WHATSAPP_COMMANDS_V990, install_whatsapp_runtime_v990 +from postmaster.whatsapp_v990.service import WhatsAppService, WhatsAppEventStore +from postmaster.whatsapp_v990.store import EncryptedAuthStore +from postmaster.webgui_whatsapp_v990 import render_whatsapp_panel, qr_renderer_status, install_webgui_whatsapp_v990 + + +class FakeMCP: + def __init__(self): self.tools={} + def remove_tool(self,name): self.tools.pop(name,None) + def add_tool(self,fn,name=None,annotations=None): self.tools[name or fn.__name__]=(fn,annotations) + + +class FakeAdapter: + def __init__(self): self.connected=False; self.calls=[] + async def start_pairing(self): self.calls.append("pair"); return {"ok":True,"qr":"qr-secret-payload","private_key":"never-return-this"} + async def reconnect(self): self.connected=True; self.calls.append("reconnect"); return {"ok":True,"connected":True} + async def send_text(self, **kwargs): self.calls.append(("text",kwargs)); return {"ok":True,"message_id":"m1"} + async def send_media(self, **kwargs): self.calls.append(("media",kwargs)); return {"ok":True,"message_id":"m2"} + async def list_groups(self): return [{"jid":"123@g.us","subject":"Test"}] + def status(self): return {"configured":True,"connected":self.connected} + + +class WhatsAppRuntimeTests(unittest.IsolatedAsyncioTestCase): + def make(self, td): + adapter=FakeAdapter() + service=WhatsAppService( + EncryptedAuthStore(str(Path(td)/"auth.db")), + WhatsAppEventStore(str(Path(td)/"events.db")), + adapter, + ) + core=SimpleNamespace(mcp=FakeMCP()) + base=SimpleNamespace() + out=install_whatsapp_runtime_v990(base,core,lambda:{"ok":True,"version_capability":"9.9.0","mcp_command_count_expected":130},service=service) + return base,core,adapter,service,out + + async def test_exact_eight_tools_and_safe_status(self): + with TemporaryDirectory() as td: + _,core,_,_,out=self.make(td) + expected={"whatsapp_status","whatsapp_start_pairing","whatsapp_reconnect","whatsapp_list_messages","whatsapp_send_text","whatsapp_send_media","whatsapp_list_groups","whatsapp_list_receipts"} + self.assertEqual(set(core.mcp.tools)-{"runtime_status"}, expected) + self.assertEqual(out["tool_count"], MCP_WHATSAPP_COMMANDS_V990) + status=core.whatsapp_status() + self.assertFalse(status["auth_store"]["private_material_exposed"]) + runtime=core.runtime_status() + self.assertTrue(runtime["whatsapp"]["read_receipts_asymmetric"]) + self.assertFalse(runtime["whatsapp"]["local_read_emits_receipt"]) + self.assertFalse(runtime["whatsapp"]["protocol_interop_verified"]) + self.assertFalse(runtime["whatsapp"]["signal_multidevice_verified"]) + self.assertFalse(runtime["whatsapp"]["controlled_account_interop_verified"]) + + async def test_default_service_wires_canonical_file_store_without_network(self): + with TemporaryDirectory() as td: + store = object() + service = WhatsAppService.create( + auth_db=str(Path(td) / "auth-real.db"), + event_db=str(Path(td) / "events-real.db"), + key_path=str(Path(td) / "auth-real.key"), + file_store=store, + file_owner_id="davide", + file_project_id="postmaster-mcp", + ) + self.assertIs(service.adapter.file_store, store) + self.assertEqual(service.adapter.file_owner_id, "davide") + self.assertEqual(service.adapter.file_project_id, "postmaster-mcp") + self.assertFalse(service.adapter.status()["connected"]) + + async def test_pairing_strips_private_material_and_persists_qr(self): + with TemporaryDirectory() as td: + _,core,_,service,_=self.make(td) + result=await core.whatsapp_start_pairing() + self.assertNotIn("private_key", result) + self.assertFalse(result["private_material_exposed"]) + self.assertEqual(service.pairing_qr(), "qr-secret-payload") + + async def test_read_does_not_receipt_but_explicit_reply_may(self): + with TemporaryDirectory() as td: + _,core,adapter,service,_=self.make(td) + service.record_incoming_message(message_id="incoming1",jid="123@s.whatsapp.net",text="hello") + listed=core.whatsapp_list_messages() + self.assertFalse(listed["read_receipt_emitted"]) + self.assertEqual(core.whatsapp_list_receipts()["receipts"], []) + sent=await core.whatsapp_send_text("123@s.whatsapp.net","reply",reply_to_message_id="incoming1") + self.assertTrue(sent["read_receipt_emitted"]) + self.assertTrue(adapter.calls[-1][1]["emit_read_receipt"]) + receipts=core.whatsapp_list_receipts()["receipts"] + self.assertEqual(receipts[0]["source"],"outbound_reply") + + async def test_media_requires_stored_file_identifier(self): + with TemporaryDirectory() as td: + _,core,adapter,_,_=self.make(td) + bad=await core.whatsapp_send_media("123@s.whatsapp.net","") + self.assertFalse(bad["ok"]) + good=await core.whatsapp_send_media("123@g.us","file-123",caption="doc") + self.assertTrue(good["ok"]) + self.assertEqual(adapter.calls[-1][1]["stored_file_id"],"file-123") + + async def test_webgui_fragment_renders_qr_locally_without_remote_service(self): + panel=render_whatsapp_panel({"paired":False,"network":{"connected":False}},qr_payload='https://wa.me/settings/linked_devices#safe-test') + qr_status=qr_renderer_status() + self.assertTrue(qr_status["local"]) + self.assertFalse(qr_status["remote_service"]) + self.assertNotIn("data-qr-payload",panel) + self.assertNotIn("wa.me/settings/linked_devices#safe-test",panel) + self.assertNotIn("