diff --git a/.github/dependabot.yml b/.github/dependabot.yml index dc5bcaed14..9ef60f3740 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,13 +6,14 @@ updates: - # "pip" is the correct setting for poetry, per https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem package-ecosystem: "pip" directory: "/" - open-pull-requests-limit: 10 + # Famedly edit. Setting open pull requests to 0 prevents new ones from opening. + open-pull-requests-limit: 0 versioning-strategy: "increase-if-necessary" schedule: interval: "weekly" # Group patch updates to packages together into a single PR, as they rarely # if ever contain breaking changes that need to be reviewed separately. - # + # # Less PRs means a streamlined review process. # # Python packages follow semantic versioning, and tend to only introduce @@ -39,7 +40,8 @@ updates: - package-ecosystem: "docker" directory: "/docker" - open-pull-requests-limit: 10 + # Famedly edit. Setting open pull requests to 0 prevents new ones from opening. + open-pull-requests-limit: 0 schedule: interval: "weekly" # For container versions, breaking changes are also typically only introduced in major @@ -57,7 +59,8 @@ updates: - package-ecosystem: "github-actions" directory: "/" - open-pull-requests-limit: 10 + # Famedly edit. Setting open pull requests to 0 prevents new ones from opening. + open-pull-requests-limit: 0 schedule: interval: "weekly" # Similarly for GitHub Actions, breaking changes are typically only introduced in major @@ -75,7 +78,8 @@ updates: - package-ecosystem: "cargo" directory: "/" - open-pull-requests-limit: 10 + # Famedly edit. Setting open pull requests to 0 prevents new ones from opening. + open-pull-requests-limit: 0 versioning-strategy: "lockfile-only" schedule: interval: "weekly" diff --git a/CHANGES.md b/CHANGES.md index 5937675dda..c3436b9968 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,48 @@ +# Synapse 1.154.0 (2026-06-04) + +No significant changes since 1.154.0rc1. + +## Famedly additions for v1.154.0_1 + +- Disabled in-repo dependabot pull requests, as this fork relies on upstream to handle dependency changes (Jason Little) + +# Synapse 1.154.0rc1 (2026-05-27) + +## Features + +- Add support for [MSC4452: Preview URL capabilities API](https://github.com/matrix-org/matrix-spec-proposals/pull/4452) which exposes a `io.element.msc4452.preview_url` capability. + If `experimental_features.msc4452_enabled` is `true`, the `/_matrix/(client/v1/media|media/v3)/preview_url` endpoint + now responds with a 403 status code when the capability is disabled. ([\#19715](https://github.com/element-hq/synapse/issues/19715)) + +## Bugfixes + +- Fix a bug in [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) that could prevent user avatars from showing if the room had an empty name. ([\#19468](https://github.com/element-hq/synapse/issues/19468), [\#19791](https://github.com/element-hq/synapse/issues/19791)) +- Fix access token cache not being invalidated for sessions using refresh tokens. Contributed by @FrenchGithubUser @ Famedly. ([\#19483](https://github.com/element-hq/synapse/issues/19483)) +- Fix bug where Synapse would return 400 (`M_BAD_JSON`) when sending a message with a `mentions` field and Synapse module `check_event_allowed` callback registered (frozen event). Contributed by @gaetan-sbt. ([\#19634](https://github.com/element-hq/synapse/issues/19634)) +- Fix long-standing but niche bug with `/sync` where it could attempt to fetch data with flawed invalid future tokens. ([\#19644](https://github.com/element-hq/synapse/issues/19644)) +- Fix `/sync` failing when [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) are enabled and the sync request filters out Ephemeral Data Units (EDUs). ([\#19787](https://github.com/element-hq/synapse/issues/19787)) +- Fix packaging for Fedora and EPEL caused by unnecessary bumping `attrs` minimum version requirement in `pyproject.toml` file. Contributed by Oleg Girko. ([\#19789](https://github.com/element-hq/synapse/issues/19789)) +- Fix merging signatures when a policy server is running under the same server name as Synapse. The bug was re-introduced in v1.153.0rc1 after being fixed earlier in v1.151.0rc1. Contributed by @tulir @ Beeper. ([\#19797](https://github.com/element-hq/synapse/issues/19797)) + +## Improved Documentation + +- Added details about how Synapse syncs the picture claim when `update_profile_information` setting is true. ([\#19508](https://github.com/element-hq/synapse/issues/19508)) + +## Internal Changes + +- Port `Event.content` field to Rust. ([\#19725](https://github.com/element-hq/synapse/issues/19725)) +- Prefer close backfill points (absolute distance). ([\#19748](https://github.com/element-hq/synapse/issues/19748)) +- Replace unique `quarantined_media` waiting patterns with standard `wait_for_stream_token(...)`. ([\#19764](https://github.com/element-hq/synapse/issues/19764)) +- Improve Synapse logging around when someone encounters `We can't get valid state history.` so you can correlate everything by `event_id`. ([\#19765](https://github.com/element-hq/synapse/issues/19765)) +- Tidy up Rust `RoomVersion` structs. ([\#19766](https://github.com/element-hq/synapse/issues/19766)) +- Update `WorkerLock` tests to better stress the `WORKER_LOCK_MAX_RETRY_INTERVAL`. ([\#19772](https://github.com/element-hq/synapse/issues/19772)) +- Refactor [MSC4242: State DAG](https://github.com/matrix-org/matrix-spec-proposals/pull/4242) checks behind a single `TypeIs` helper to avoid scattered `isinstance` casts. ([\#19774](https://github.com/element-hq/synapse/issues/19774)) +- Use `StrCollection` for `prev_state_events`. ([\#19777](https://github.com/element-hq/synapse/issues/19777)) +- Fix up the construction of events in tests, ahead of the Rust event port. ([\#19781](https://github.com/element-hq/synapse/issues/19781)) + + + + # Synapse 1.153.0 (2026-05-19) No significant changes since 1.153.0rc3. diff --git a/debian/changelog b/debian/changelog index b1a4e04bc3..7750ca563c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +matrix-synapse-py3 (1.154.0) stable; urgency=medium + + * New Synapse release 1.154.0. + + -- Synapse Packaging team Thu, 04 Jun 2026 14:16:23 +0100 + +matrix-synapse-py3 (1.154.0~rc1) stable; urgency=medium + + * New Synapse release 1.154.0rc1. + + -- Synapse Packaging team Wed, 27 May 2026 12:23:54 +0100 + matrix-synapse-py3 (1.153.0) stable; urgency=medium * New Synapse release 1.153.0. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index ba398322a7..548687a39e 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3822,7 +3822,10 @@ This setting has the following sub-options: Defaults to `null`. -* `update_profile_information` (boolean): Use this setting to keep a user's profile fields in sync with information from the identity provider. Currently only syncing the displayname is supported. Fields are checked on every SSO login, and are updated if necessary. Note that enabling this option will override user profile information, regardless of whether users have opted-out of syncing that information when first signing in. Defaults to `false`. +* `update_profile_information` (boolean): Use this setting to keep a user's profile fields in sync with information from the identity provider. Fields are checked on every SSO login, and are updated if necessary. Note that enabling this option will override user profile information, regardless of whether users have opted-out of syncing that information when first signing in. Fields that will be synced: + * displayname + * picture - only if Synapse media repository is running in the main + process (i.e. not workerized) and media is stored locally Defaults to `false`. Example configuration: ```yaml diff --git a/poetry.lock b/poetry.lock index 6a8870590d..75b56469ee 100644 --- a/poetry.lock +++ b/poetry.lock @@ -26,15 +26,15 @@ files = [ [[package]] name = "authlib" -version = "1.6.11" +version = "1.6.12" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = true python-versions = ">=3.9" groups = ["main"] markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ - {file = "authlib-1.6.11-py2.py3-none-any.whl", hash = "sha256:c8687a9a26451c51a34a06fa17bb97cb15bba46a6a626755e2d7f50da8bff3e3"}, - {file = "authlib-1.6.11.tar.gz", hash = "sha256:64db35b9b01aeccb4715a6c9a6613a06f2bd7be2ab9d2eb89edd1dfc7580a38f"}, + {file = "authlib-1.6.12-py2.py3-none-any.whl", hash = "sha256:e9229ad7fde610b139dd12f5edbe97eab9ee78bfb85691247e767727850b99ab"}, + {file = "authlib-1.6.12.tar.gz", hash = "sha256:0656d8482f28fc8221929d5f35b2bde5d13e10555ebc06b4561b0d622e83b1bd"}, ] [package.dependencies] @@ -582,14 +582,14 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.47" +version = "3.1.50" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "gitpython-3.1.47-py3-none-any.whl", hash = "sha256:489f590edfd6d20571b2c0e72c6a6ac6915ee8b8cd04572330e3842207a78905"}, - {file = "gitpython-3.1.47.tar.gz", hash = "sha256:dba27f922bd2b42cb54c87a8ab3cb6beb6bf07f3d564e21ac848913a05a8a3cd"}, + {file = "gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9"}, + {file = "gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc"}, ] [package.dependencies] @@ -1331,153 +1331,147 @@ files = [ [[package]] name = "lxml" -version = "6.0.2" +version = "6.1.0" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." optional = true python-versions = ">=3.8" groups = ["main"] markers = "extra == \"url-preview\" or extra == \"all\"" files = [ - {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388"}, - {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f952dacaa552f3bb8834908dddd500ba7d508e6ea6eb8c52eb2d28f48ca06a31"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:71695772df6acea9f3c0e59e44ba8ac50c4f125217e84aab21074a1a55e7e5c9"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:17f68764f35fd78d7c4cc4ef209a184c38b65440378013d24b8aecd327c3e0c8"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:058027e261afed589eddcfe530fcc6f3402d7fd7e89bfd0532df82ebc1563dba"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8ffaeec5dfea5881d4c9d8913a32d10cfe3923495386106e4a24d45300ef79c"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:f2e3b1a6bb38de0bc713edd4d612969dd250ca8b724be8d460001a387507021c"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d6690ec5ec1cce0385cb20896b16be35247ac8c2046e493d03232f1c2414d321"}, - {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f2a50c3c1d11cad0ebebbac357a97b26aa79d2bcaf46f256551152aa85d3a4d1"}, - {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:3efe1b21c7801ffa29a1112fab3b0f643628c30472d507f39544fd48e9549e34"}, - {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:59c45e125140b2c4b33920d21d83681940ca29f0b83f8629ea1a2196dc8cfe6a"}, - {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:452b899faa64f1805943ec1c0c9ebeaece01a1af83e130b69cdefeda180bb42c"}, - {file = "lxml-6.0.2-cp310-cp310-win32.whl", hash = "sha256:1e786a464c191ca43b133906c6903a7e4d56bef376b75d97ccbb8ec5cf1f0a4b"}, - {file = "lxml-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:dacf3c64ef3f7440e3167aa4b49aa9e0fb99e0aa4f9ff03795640bf94531bcb0"}, - {file = "lxml-6.0.2-cp310-cp310-win_arm64.whl", hash = "sha256:45f93e6f75123f88d7f0cfd90f2d05f441b808562bf0bc01070a00f53f5028b5"}, - {file = "lxml-6.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:13e35cbc684aadf05d8711a5d1b5857c92e5e580efa9a0d2be197199c8def607"}, - {file = "lxml-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b1675e096e17c6fe9c0e8c81434f5736c0739ff9ac6123c87c2d452f48fc938"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac6e5811ae2870953390452e3476694196f98d447573234592d30488147404d"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5aa0fc67ae19d7a64c3fe725dc9a1bb11f80e01f78289d05c6f62545affec438"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de496365750cc472b4e7902a485d3f152ecf57bd3ba03ddd5578ed8ceb4c5964"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:200069a593c5e40b8f6fc0d84d86d970ba43138c3e68619ffa234bc9bb806a4d"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d2de809c2ee3b888b59f995625385f74629707c9355e0ff856445cdcae682b7"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:b2c3da8d93cf5db60e8858c17684c47d01fee6405e554fb55018dd85fc23b178"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:442de7530296ef5e188373a1ea5789a46ce90c4847e597856570439621d9c553"}, - {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2593c77efde7bfea7f6389f1ab249b15ed4aa5bc5cb5131faa3b843c429fbedb"}, - {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3e3cb08855967a20f553ff32d147e14329b3ae70ced6edc2f282b94afbc74b2a"}, - {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2ed6c667fcbb8c19c6791bbf40b7268ef8ddf5a96940ba9404b9f9a304832f6c"}, - {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b8f18914faec94132e5b91e69d76a5c1d7b0c73e2489ea8929c4aaa10b76bbf7"}, - {file = "lxml-6.0.2-cp311-cp311-win32.whl", hash = "sha256:6605c604e6daa9e0d7f0a2137bdc47a2e93b59c60a65466353e37f8272f47c46"}, - {file = "lxml-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e5867f2651016a3afd8dd2c8238baa66f1e2802f44bc17e236f547ace6647078"}, - {file = "lxml-6.0.2-cp311-cp311-win_arm64.whl", hash = "sha256:4197fb2534ee05fd3e7afaab5d8bfd6c2e186f65ea7f9cd6a82809c887bd1285"}, - {file = "lxml-6.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a59f5448ba2ceccd06995c95ea59a7674a10de0810f2ce90c9006f3cbc044456"}, - {file = "lxml-6.0.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e8113639f3296706fbac34a30813929e29247718e88173ad849f57ca59754924"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a8bef9b9825fa8bc816a6e641bb67219489229ebc648be422af695f6e7a4fa7f"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:65ea18d710fd14e0186c2f973dc60bb52039a275f82d3c44a0e42b43440ea534"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c371aa98126a0d4c739ca93ceffa0fd7a5d732e3ac66a46e74339acd4d334564"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:700efd30c0fa1a3581d80a748157397559396090a51d306ea59a70020223d16f"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c33e66d44fe60e72397b487ee92e01da0d09ba2d66df8eae42d77b6d06e5eba0"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90a345bbeaf9d0587a3aaffb7006aa39ccb6ff0e96a57286c0cb2fd1520ea192"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:064fdadaf7a21af3ed1dcaa106b854077fbeada827c18f72aec9346847cd65d0"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbc74f42c3525ac4ffa4b89cbdd00057b6196bcefe8bce794abd42d33a018092"}, - {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6ddff43f702905a4e32bc24f3f2e2edfe0f8fde3277d481bffb709a4cced7a1f"}, - {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6da5185951d72e6f5352166e3da7b0dc27aa70bd1090b0eb3f7f7212b53f1bb8"}, - {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:57a86e1ebb4020a38d295c04fc79603c7899e0df71588043eb218722dabc087f"}, - {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2047d8234fe735ab77802ce5f2297e410ff40f5238aec569ad7c8e163d7b19a6"}, - {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f91fd2b2ea15a6800c8e24418c0775a1694eefc011392da73bc6cef2623b322"}, - {file = "lxml-6.0.2-cp312-cp312-win32.whl", hash = "sha256:3ae2ce7d6fedfb3414a2b6c5e20b249c4c607f72cb8d2bb7cc9c6ec7c6f4e849"}, - {file = "lxml-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:72c87e5ee4e58a8354fb9c7c84cbf95a1c8236c127a5d1b7683f04bed8361e1f"}, - {file = "lxml-6.0.2-cp312-cp312-win_arm64.whl", hash = "sha256:61cb10eeb95570153e0c0e554f58df92ecf5109f75eacad4a95baa709e26c3d6"}, - {file = "lxml-6.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9b33d21594afab46f37ae58dfadd06636f154923c4e8a4d754b0127554eb2e77"}, - {file = "lxml-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6c8963287d7a4c5c9a432ff487c52e9c5618667179c18a204bdedb27310f022f"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1941354d92699fb5ffe6ed7b32f9649e43c2feb4b97205f75866f7d21aa91452"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb2f6ca0ae2d983ded09357b84af659c954722bbf04dea98030064996d156048"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb2a12d704f180a902d7fa778c6d71f36ceb7b0d317f34cdc76a5d05aa1dd1df"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6ec0e3f745021bfed19c456647f0298d60a24c9ff86d9d051f52b509663feeb1"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:846ae9a12d54e368933b9759052d6206a9e8b250291109c48e350c1f1f49d916"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef9266d2aa545d7374938fb5c484531ef5a2ec7f2d573e62f8ce722c735685fd"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:4077b7c79f31755df33b795dc12119cb557a0106bfdab0d2c2d97bd3cf3dffa6"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a7c5d5e5f1081955358533be077166ee97ed2571d6a66bdba6ec2f609a715d1a"}, - {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8f8d0cbd0674ee89863a523e6994ac25fd5be9c8486acfc3e5ccea679bad2679"}, - {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:2cbcbf6d6e924c28f04a43f3b6f6e272312a090f269eff68a2982e13e5d57659"}, - {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:dfb874cfa53340009af6bdd7e54ebc0d21012a60a4e65d927c2e477112e63484"}, - {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fb8dae0b6b8b7f9e96c26fdd8121522ce5de9bb5538010870bd538683d30e9a2"}, - {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:358d9adae670b63e95bc59747c72f4dc97c9ec58881d4627fe0120da0f90d314"}, - {file = "lxml-6.0.2-cp313-cp313-win32.whl", hash = "sha256:e8cd2415f372e7e5a789d743d133ae474290a90b9023197fd78f32e2dc6873e2"}, - {file = "lxml-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:b30d46379644fbfc3ab81f8f82ae4de55179414651f110a1514f0b1f8f6cb2d7"}, - {file = "lxml-6.0.2-cp313-cp313-win_arm64.whl", hash = "sha256:13dcecc9946dca97b11b7c40d29fba63b55ab4170d3c0cf8c0c164343b9bfdcf"}, - {file = "lxml-6.0.2-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:b0c732aa23de8f8aec23f4b580d1e52905ef468afb4abeafd3fec77042abb6fe"}, - {file = "lxml-6.0.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4468e3b83e10e0317a89a33d28f7aeba1caa4d1a6fd457d115dd4ffe90c5931d"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:abd44571493973bad4598a3be7e1d807ed45aa2adaf7ab92ab7c62609569b17d"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:370cd78d5855cfbffd57c422851f7d3864e6ae72d0da615fca4dad8c45d375a5"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:901e3b4219fa04ef766885fb40fa516a71662a4c61b80c94d25336b4934b71c0"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:a4bf42d2e4cf52c28cc1812d62426b9503cdb0c87a6de81442626aa7d69707ba"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2c7fdaa4d7c3d886a42534adec7cfac73860b89b4e5298752f60aa5984641a0"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98a5e1660dc7de2200b00d53fa00bcd3c35a3608c305d45a7bbcaf29fa16e83d"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:dc051506c30b609238d79eda75ee9cab3e520570ec8219844a72a46020901e37"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8799481bbdd212470d17513a54d568f44416db01250f49449647b5ab5b5dccb9"}, - {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9261bb77c2dab42f3ecd9103951aeca2c40277701eb7e912c545c1b16e0e4917"}, - {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:65ac4a01aba353cfa6d5725b95d7aed6356ddc0a3cd734de00124d285b04b64f"}, - {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:b22a07cbb82fea98f8a2fd814f3d1811ff9ed76d0fc6abc84eb21527596e7cc8"}, - {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d759cdd7f3e055d6bc8d9bec3ad905227b2e4c785dc16c372eb5b5e83123f48a"}, - {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:945da35a48d193d27c188037a05fec5492937f66fb1958c24fc761fb9d40d43c"}, - {file = "lxml-6.0.2-cp314-cp314-win32.whl", hash = "sha256:be3aaa60da67e6153eb15715cc2e19091af5dc75faef8b8a585aea372507384b"}, - {file = "lxml-6.0.2-cp314-cp314-win_amd64.whl", hash = "sha256:fa25afbadead523f7001caf0c2382afd272c315a033a7b06336da2637d92d6ed"}, - {file = "lxml-6.0.2-cp314-cp314-win_arm64.whl", hash = "sha256:063eccf89df5b24e361b123e257e437f9e9878f425ee9aae3144c77faf6da6d8"}, - {file = "lxml-6.0.2-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:6162a86d86893d63084faaf4ff937b3daea233e3682fb4474db07395794fa80d"}, - {file = "lxml-6.0.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:414aaa94e974e23a3e92e7ca5b97d10c0cf37b6481f50911032c69eeb3991bba"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48461bd21625458dd01e14e2c38dd0aea69addc3c4f960c30d9f59d7f93be601"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:25fcc59afc57d527cfc78a58f40ab4c9b8fd096a9a3f964d2781ffb6eb33f4ed"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5179c60288204e6ddde3f774a93350177e08876eaf3ab78aa3a3649d43eb7d37"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:967aab75434de148ec80597b75062d8123cadf2943fb4281f385141e18b21338"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d100fcc8930d697c6561156c6810ab4a508fb264c8b6779e6e61e2ed5e7558f9"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ca59e7e13e5981175b8b3e4ab84d7da57993eeff53c07764dcebda0d0e64ecd"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:957448ac63a42e2e49531b9d6c0fa449a1970dbc32467aaad46f11545be9af1d"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b7fc49c37f1786284b12af63152fe1d0990722497e2d5817acfe7a877522f9a9"}, - {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e19e0643cc936a22e837f79d01a550678da8377d7d801a14487c10c34ee49c7e"}, - {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:1db01e5cf14345628e0cbe71067204db658e2fb8e51e7f33631f5f4735fefd8d"}, - {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:875c6b5ab39ad5291588aed6925fac99d0097af0dd62f33c7b43736043d4a2ec"}, - {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cdcbed9ad19da81c480dfd6dd161886db6096083c9938ead313d94b30aadf272"}, - {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:80dadc234ebc532e09be1975ff538d154a7fa61ea5031c03d25178855544728f"}, - {file = "lxml-6.0.2-cp314-cp314t-win32.whl", hash = "sha256:da08e7bb297b04e893d91087df19638dc7a6bb858a954b0cc2b9f5053c922312"}, - {file = "lxml-6.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:252a22982dca42f6155125ac76d3432e548a7625d56f5a273ee78a5057216eca"}, - {file = "lxml-6.0.2-cp314-cp314t-win_arm64.whl", hash = "sha256:bb4c1847b303835d89d785a18801a883436cdfd5dc3d62947f9c49e24f0f5a2c"}, - {file = "lxml-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a656ca105115f6b766bba324f23a67914d9c728dafec57638e2b92a9dcd76c62"}, - {file = "lxml-6.0.2-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c54d83a2188a10ebdba573f16bd97135d06c9ef60c3dc495315c7a28c80a263f"}, - {file = "lxml-6.0.2-cp38-cp38-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:1ea99340b3c729beea786f78c38f60f4795622f36e305d9c9be402201efdc3b7"}, - {file = "lxml-6.0.2-cp38-cp38-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:af85529ae8d2a453feee4c780d9406a5e3b17cee0dd75c18bd31adcd584debc3"}, - {file = "lxml-6.0.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:fe659f6b5d10fb5a17f00a50eb903eb277a71ee35df4615db573c069bcf967ac"}, - {file = "lxml-6.0.2-cp38-cp38-win32.whl", hash = "sha256:5921d924aa5468c939d95c9814fa9f9b5935a6ff4e679e26aaf2951f74043512"}, - {file = "lxml-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:0aa7070978f893954008ab73bb9e3c24a7c56c054e00566a21b553dc18105fca"}, - {file = "lxml-6.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2c8458c2cdd29589a8367c09c8f030f1d202be673f0ca224ec18590b3b9fb694"}, - {file = "lxml-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3fee0851639d06276e6b387f1c190eb9d7f06f7f53514e966b26bae46481ec90"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b2142a376b40b6736dfc214fd2902409e9e3857eff554fed2d3c60f097e62a62"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a6b5b39cc7e2998f968f05309e666103b53e2edd01df8dc51b90d734c0825444"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4aec24d6b72ee457ec665344a29acb2d35937d5192faebe429ea02633151aad"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:b42f4d86b451c2f9d06ffb4f8bbc776e04df3ba070b9fe2657804b1b40277c48"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6cdaefac66e8b8f30e37a9b4768a391e1f8a16a7526d5bc77a7928408ef68e93"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:b738f7e648735714bbb82bdfd030203360cfeab7f6e8a34772b3c8c8b820568c"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daf42de090d59db025af61ce6bdb2521f0f102ea0e6ea310f13c17610a97da4c"}, - {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:66328dabea70b5ba7e53d94aa774b733cf66686535f3bc9250a7aab53a91caaf"}, - {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:e237b807d68a61fc3b1e845407e27e5eb8ef69bc93fe8505337c1acb4ee300b6"}, - {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:ac02dc29fd397608f8eb15ac1610ae2f2f0154b03f631e6d724d9e2ad4ee2c84"}, - {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:817ef43a0c0b4a77bd166dc9a09a555394105ff3374777ad41f453526e37f9cb"}, - {file = "lxml-6.0.2-cp39-cp39-win32.whl", hash = "sha256:bc532422ff26b304cfb62b328826bd995c96154ffd2bac4544f37dbb95ecaa8f"}, - {file = "lxml-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:995e783eb0374c120f528f807443ad5a83a656a8624c467ea73781fc5f8a8304"}, - {file = "lxml-6.0.2-cp39-cp39-win_arm64.whl", hash = "sha256:08b9d5e803c2e4725ae9e8559ee880e5328ed61aa0935244e0515d7d9dbec0aa"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e748d4cf8fef2526bb2a589a417eba0c8674e29ffcb570ce2ceca44f1e567bf6"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4ddb1049fa0579d0cbd00503ad8c58b9ab34d1254c77bc6a5576d96ec7853dba"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cb233f9c95f83707dae461b12b720c1af9c28c2d19208e1be03387222151daf5"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc456d04db0515ce3320d714a1eac7a97774ff0849e7718b492d957da4631dd4"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2613e67de13d619fd283d58bda40bff0ee07739f624ffee8b13b631abf33083d"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:24a8e756c982c001ca8d59e87c80c4d9dcd4d9b44a4cbeb8d9be4482c514d41d"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1c06035eafa8404b5cf475bb37a9f6088b0aca288d4ccc9d69389750d5543700"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c7d13103045de1bdd6fe5d61802565f1a3537d70cd3abf596aa0af62761921ee"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a3c150a95fbe5ac91de323aa756219ef9cf7fde5a3f00e2281e30f33fa5fa4f"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60fa43be34f78bebb27812ed90f1925ec99560b0fa1decdb7d12b84d857d31e9"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21c73b476d3cfe836be731225ec3421fa2f048d84f6df6a8e70433dff1376d5a"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:27220da5be049e936c3aca06f174e8827ca6445a4353a1995584311487fc4e3e"}, - {file = "lxml-6.0.2.tar.gz", hash = "sha256:cd79f3367bd74b317dda655dc8fcfa304d9eb6e4fb06b7168c5cf27f96e0cd62"}, + {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:41dcc4c7b10484257cbd6c37b83ddb26df2b0e5aff5ac00d095689015af868ec"}, + {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a31286dbb5e74c8e9a5344465b77ab4c5bd511a253b355b5ca2fae7e579fafec"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1bc4cc83fb7f66ffb16f74d6dd0162e144333fc36ebcce32246f80c8735b2551"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:20cf4d0651987c906a2f5cba4e3a8d6ba4bfdf973cfe2a96c0d6053888ea2ecd"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffb34ea45a82dd637c2c97ae1bbb920850c1e59bcae79ce1c15af531d83e7215"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1d9b99e5b2597e4f5aed2484fef835256fa1b68a19e4265c97628ef4bf8bcf4"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux_2_28_i686.whl", hash = "sha256:d43aa26dcda363f21e79afa0668f5029ed7394b3bb8c92a6927a3d34e8b610ea"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:6262b87f9e5c1e5fe501d6c153247289af42eb44ad7660b9b3de17baaf92d6f6"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d1392c569c032f78a11a25d1de1c43fff13294c793b39e19d84fade3045cbbc3"}, + {file = "lxml-6.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:045e387d1f4f42a418380930fa3f45c73c9b392faf67e495e58902e68e8f44a7"}, + {file = "lxml-6.1.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:9f93d5b8b07f73e8c77e3c6556a3db269918390c804b5e5fcdd4858232cc8f16"}, + {file = "lxml-6.1.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:de550d129f18d8ab819651ffe4f38b1b713c7e116707de3c0c6400d0ef34fbc1"}, + {file = "lxml-6.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c08da09dc003c9e8c70e06b53a11db6fb3b250c21c4236b03c7d7b443c318e7a"}, + {file = "lxml-6.1.0-cp310-cp310-win32.whl", hash = "sha256:37448bf9c7d7adfc5254763901e2bbd6bb876228dfc1fc7f66e58c06368a7544"}, + {file = "lxml-6.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:2593a0a6621545b9095b71ad74ed4226eba438a7d9fc3712a99bdb15508cf93a"}, + {file = "lxml-6.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:e80807d72f96b96ad5588cb85c75616e4f2795a7737d4630784c51497beb7776"}, + {file = "lxml-6.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cec05be8c876f92a5aa07b01d60bbb4d11cfbdd654cad0561c0d7b5c043a61b9"}, + {file = "lxml-6.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9c03e048b6ce8e77b09c734e931584894ecd58d08296804ca2d0b184c933ce50"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:942454ff253da14218f972b23dc72fa4edf6c943f37edd19cd697618b626fac5"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d036ee7b99d5148072ac7c9b847193decdfeac633db350363f7bce4fff108f0e"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ae5d8d5427f3cc317e7950f2da7ad276df0cfa37b8de2f5658959e618ea8512"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:363e47283bde87051b821826e71dde47f107e08614e1aa312ba0c5711e77738c"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux_2_28_i686.whl", hash = "sha256:f504d861d9f2a8f94020130adac88d66de93841707a23a86244263d1e54682f5"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:23a5dc68e08ed13331d61815c08f260f46b4a60fdd1640bbeb82cf89a9d90289"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f15401d8d3dbf239e23c818afc10c7207f7b95f9a307e092122b6f86dd43209a"}, + {file = "lxml-6.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fcf3da95e93349e0647d48d4b36a12783105bcc74cb0c416952f9988410846a3"}, + {file = "lxml-6.1.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:0d082495c5fcf426e425a6e28daaba1fcb6d8f854a4ff01effb1f1f381203eb9"}, + {file = "lxml-6.1.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:e3c4f84b24a1fcba435157d111c4b755099c6ff00a3daee1ad281817de75ed11"}, + {file = "lxml-6.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:976a6b39b1b13e8c354ad8d3f261f3a4ac6609518af91bdb5094760a08f132c4"}, + {file = "lxml-6.1.0-cp311-cp311-win32.whl", hash = "sha256:857efde87d365706590847b916baff69c0bc9252dc5af030e378c9800c0b10e3"}, + {file = "lxml-6.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:183bfb45a493081943be7ea2b5adfc2b611e1cf377cefa8b8a8be404f45ef9a7"}, + {file = "lxml-6.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:19f4164243fc206d12ed3d866e80e74f5bc3627966520da1a5f97e42c32a3f39"}, + {file = "lxml-6.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d2f17a16cd8751e8eb233a7e41aecdf8e511712e00088bf9be455f604cd0d28d"}, + {file = "lxml-6.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f0cea5b1d3e6e77d71bd2b9972eb2446221a69dc52bb0b9c3c6f6e5700592d93"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fc46da94826188ed45cb53bd8e3fc076ae22675aea2087843d4735627f867c6d"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9147d8e386ec3b82c3b15d88927f734f565b0aaadef7def562b853adca45784a"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5715e0e28736a070f3f34a7ccc09e2fdcba0e3060abbcf61a1a5718ff6d6b105"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4937460dc5df0cdd2f06a86c285c28afda06aefa3af949f9477d3e8df430c485"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc783ee3147e60a25aa0445ea82b3e8aabb83b240f2b95d32cb75587ff781814"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:40d9189f80075f2e1f88db21ef815a2b17b28adf8e50aaf5c789bfe737027f32"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:05b9b8787e35bec69e68daf4952b2e6dfcfb0db7ecf1a06f8cdfbbac4eb71aad"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0f0f08beb0182e3e9a86fae124b3c47a7b41b7b69b225e1377db983802404e54"}, + {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73becf6d8c81d4c76b1014dbd3584cb26d904492dcf73ca85dc8bff08dcd6d2d"}, + {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1ae225f66e5938f4fa29d37e009a3bb3b13032ac57eb4eb42afa44f6e4054e69"}, + {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:690022c7fae793b0489aa68a658822cea83e0d5933781811cabbf5ea3bcfe73d"}, + {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:63aeafc26aac0be8aff14af7871249e87ea1319be92090bfd632ec68e03b16a5"}, + {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:264c605ab9c0e4aa1a679636f4582c4d3313700009fac3ec9c3412ed0d8f3e1d"}, + {file = "lxml-6.1.0-cp312-cp312-win32.whl", hash = "sha256:56971379bc5ee8037c5a0f09fa88f66cdb7d37c3e38af3e45cf539f41131ac1f"}, + {file = "lxml-6.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:bba078de0031c219e5dd06cf3e6bf8fb8e6e64a77819b358f53bb132e3e03366"}, + {file = "lxml-6.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:c3592631e652afa34999a088f98ba7dfc7d6aff0d535c410bea77a71743f3819"}, + {file = "lxml-6.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a0092f2b107b69601adf562a57c956fbb596e05e3e6651cabd3054113b007e45"}, + {file = "lxml-6.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fc7140d7a7386e6b545d41b7358f4d02b656d4053f5fa6859f92f4b9c2572c4d"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:419c58fc92cc3a2c3fa5f78c63dbf5da70c1fa9c1b25f25727ecee89a96c7de2"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:37fabd1452852636cf38ecdcc9dd5ca4bba7a35d6c53fa09725deeb894a87491"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2853c8b2170cc6cd54a6b4d50d2c1a8a7aeca201f23804b4898525c7a152cfc"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8e369cbd690e788c8d15e56222d91a09c6a417f49cbc543040cba0fe2e25a79e"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e69aa6805905807186eb00e66c6d97a935c928275182eb02ee40ba00da9623b2"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:4bd1bdb8a9e0e2dd229de19b5f8aebac80e916921b4b2c6ef8a52bc131d0c1f9"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:cbd7b79cdcb4986ad78a2662625882747f09db5e4cd7b2ae178a88c9c51b3dfe"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:43e4d297f11080ec9d64a4b1ad7ac02b4484c9f0e2179d9c4ef78e886e747b88"}, + {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cc16682cc987a3da00aa56a3aa3075b08edb10d9b1e476938cfdbee8f3b67181"}, + {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:d6d8efe71429635f0559579092bb5e60560d7b9115ee38c4adbea35632e7fa24"}, + {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7e39ab3a28af7784e206d8606ec0e4bcad0190f63a492bca95e94e5a4aef7f6e"}, + {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9eb667bf50856c4a58145f8ca2d5e5be160191e79eb9e30855a476191b3c3495"}, + {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7f4a77d6f7edf9230cee3e1f7f6764722a41604ee5681844f18db9a81ea0ec33"}, + {file = "lxml-6.1.0-cp313-cp313-win32.whl", hash = "sha256:28902146ffbe5222df411c5d19e5352490122e14447e98cd118907ee3fd6ee62"}, + {file = "lxml-6.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:4a1503c56e4e2b38dc76f2f2da7bae69670c0f1933e27cfa34b2fa5876410b16"}, + {file = "lxml-6.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:e0af85773850417d994d019741239b901b22c6680206f46a34766926e466141d"}, + {file = "lxml-6.1.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:ab863fd37458fed6456525f297d21239d987800c46e67da5ef04fc6b3dd93ac8"}, + {file = "lxml-6.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6fd8b1df8254ff4fd93fd31da1fc15770bde23ac045be9bb1f87425702f61cc9"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:47024feaae386a92a146af0d2aeed65229bf6fff738e6a11dda6b0015fb8fd03"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3f00972f84450204cd5d93a5395965e348956aaceaadec693a22ec743f8ae3eb"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97faa0860e13b05b15a51fb4986421ef7a30f0b3334061c416e0981e9450ca4c"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:972a6451204798675407beaad97b868d0c733d9a74dafefc63120b81b8c2de28"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fe022f20bc4569ec66b63b3fb275a3d628d9d32da6326b2982584104db6d3086"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:75c4c7c619a744f972f4451bf5adf6d0fb00992a1ffc9fd78e13b0bc817cc99f"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3648f20d25102a22b6061c688beb3a805099ea4beb0a01ce62975d926944d292"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:77b9f99b17cbf14026d1e618035077060fc7195dd940d025149f3e2e830fbfcb"}, + {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:32662519149fd7a9db354175aa5e417d83485a8039b8aaa62f873ceee7ea4cad"}, + {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:73d658216fc173cf2c939e90e07b941c5e12736b0bf6a99e7af95459cfe8eabb"}, + {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ac4db068889f8772a4a698c5980ec302771bb545e10c4b095d4c8be26749616f"}, + {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:45e9dfbd1b661eb64ba0d4dbe762bd210c42d86dd1e5bd2bdf89d634231beb43"}, + {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:89e8d73d09ac696a5ba42ec69787913d53284f12092f651506779314f10ba585"}, + {file = "lxml-6.1.0-cp314-cp314-win32.whl", hash = "sha256:ebe33f4ec1b2de38ceb225a1749a2965855bffeef435ba93cd2d5d540783bf2f"}, + {file = "lxml-6.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:398443df51c538bd578529aa7e5f7afc6c292644174b47961f3bf87fe5741120"}, + {file = "lxml-6.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:8c8984e1d8c4b3949e419158fda14d921ff703a9ed8a47236c6eb7a2b6cb4946"}, + {file = "lxml-6.1.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:1081dd10bc6fa437db2500e13993abf7cc30716d0a2f40e65abb935f02ec559c"}, + {file = "lxml-6.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:dabecc48db5f42ba348d1f5d5afdc54c6c4cc758e676926c7cd327045749517d"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e3dd5fe19c9e0ac818a9c7f132a5e43c1339ec1cbbfecb1a938bd3a47875b7c9"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9e7b0a4ca6dcc007a4cef00a761bba2dea959de4bd2df98f926b33c92ca5dfb9"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d27bbe326c6b539c64b42638b18bc6003a8d88f76213a97ac9ed4f885efeab7"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4e425db0c5445ef0ad56b0eec54f89b88b2d884656e536a90b2f52aecb4ca86"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b89b098105b8599dc57adac95d1813409ac476d3c948a498775d3d0c6124bfb"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:c4a699432846df86cc3de502ee85f445ebad748a1c6021d445f3e514d2cd4b1c"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:30e7b2ed63b6c8e97cca8af048589a788ab5c9c905f36d9cf1c2bb549f450d2f"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:022981127642fe19866d2907d76241bb07ed21749601f727d5d5dd1ce5d1b773"}, + {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:23cad0cc86046d4222f7f418910e46b89971c5a45d3c8abfad0f64b7b05e4a9b"}, + {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:21c3302068f50d1e8728c67c87ba92aa87043abee517aa2576cca1855326b405"}, + {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:be10838781cb3be19251e276910cd508fe127e27c3242e50521521a0f3781690"}, + {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:2173a7bffe97667bbf0767f8a99e587740a8c56fdf3befac4b09cb29a80276fd"}, + {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c6854e9cf99c84beb004eecd7d3a3868ef1109bf2b1df92d7bc11e96a36c2180"}, + {file = "lxml-6.1.0-cp314-cp314t-win32.whl", hash = "sha256:00750d63ef0031a05331b9223463b1c7c02b9004cef2346a5b2877f0f9494dd2"}, + {file = "lxml-6.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:80410c3a7e3c617af04de17caa9f9f20adaa817093293d69eae7d7d0522836f5"}, + {file = "lxml-6.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:26dd9f57ee3bd41e7d35b4c98a2ffd89ed11591649f421f0ec19f67d50ec67ac"}, + {file = "lxml-6.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b6c2f225662bc5ad416bdd06f72ca301b31b39ce4261f0e0097017fc2891b940"}, + {file = "lxml-6.1.0-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a86f06f059e22a0d574990ee2df24ede03f7f3c68c1336293eee9536c4c776cd"}, + {file = "lxml-6.1.0-cp38-cp38-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:468479e52ecf3ec23799c863336d02c05fc2f7ffd1a1424eeeb9a28d4eb69d13"}, + {file = "lxml-6.1.0-cp38-cp38-manylinux_2_28_i686.whl", hash = "sha256:a02ca8fe48815bddcfca3248efe54451abb9dbf2f7d1c5744c8aa4142d476919"}, + {file = "lxml-6.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:bb40648d96157f9081886defe13eac99253e663be969ff938a9289eff6e47b72"}, + {file = "lxml-6.1.0-cp38-cp38-win32.whl", hash = "sha256:1dd6a1c3ad4cb674f44525d9957f3e9c209bb6dd9213245195167a281fcc2bdc"}, + {file = "lxml-6.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:4e2c54d6b47361d0f1d3bc8d4e082ad87201e56ccdcca4d3b9ee3644ff595ec8"}, + {file = "lxml-6.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:920354904d1cb86577d4b3cfe2830c2dbe81d6f4449e57ada428f1609b5985f7"}, + {file = "lxml-6.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c871299c595ee004d186f61840f0bfc4941aa3f17c8ba4a565ead7e4f4f820ee"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d0d799ff958655781296ec870d5e2448e75150da2b3d07f13ff5b0c2c35beefd"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ba11752e346bd804ea312ec2eea2532dfa8b8d3261d81a32ef9e6ab16256280"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26c5272c6a4bf4cf32d3f5a7890c942b0e04438691157d341616d02cca74d4bd"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c53fa3a5a52122d590e847a57ccf955557b9634a7f99ff5a35131321b0a85317"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux_2_28_i686.whl", hash = "sha256:76b958b4ea3104483c20f74866d55aa056546e15ebe83dd7aecd63698f43b755"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:8c11b984b5ce6add4dccc7144c7be5d364d298f15b0c6a57da1991baedc750ce"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d3829a6e6fd550a219564912d4002c537f65da4c6ae4e093cc34462f4fa027ad"}, + {file = "lxml-6.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:52b0ac6903cf74ebf997eb8c682d2fbac7d1ab7e4c552413eec55868a9b73f39"}, + {file = "lxml-6.1.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:29f5c00cb7d752bce2c70ebd2d31b0a42f9499ffdd3ecb2f31a5b73ee43031ad"}, + {file = "lxml-6.1.0-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:c748ebcb6877de89f48ab90ca96642ac458fff5dec291a2b9337cd4d0934e383"}, + {file = "lxml-6.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:08950a23f296b3f83521577274e3d3b0f3d739bf2e68d01a752e4288bc50d286"}, + {file = "lxml-6.1.0-cp39-cp39-win32.whl", hash = "sha256:11a873c77a181b4fef9c2e357d08ed399542c2af1390101da66720a19c7c9618"}, + {file = "lxml-6.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:81ff55c70b67d19d52b6fd118a114c0a4c97d799cd3089ff9bd9e2ff4b414ee2"}, + {file = "lxml-6.1.0-cp39-cp39-win_arm64.whl", hash = "sha256:481d6e2104285d9add34f41b42b247b76b61c5b5c26c303c2e9707bbf8bd9a64"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:546b66c0dd1bb8d9fa89d7123e5fa19a8aff3a1f2141eb22df96112afb17b842"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5cfa1a34df366d9dc0d5eaf420f4cf2bb1e1bebe1066d1c2fc28c179f8a4004c"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db88156fcf544cdbf0d95588051515cfdfd4c876fc66444eb98bceb5d6db76de"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:07f98f5496f96bf724b1e3c933c107f0cbf2745db18c03d2e13a291c3afd2635"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4642e04449a1e164b5ff71ffd901ddb772dfabf5c9adf1b7be5dffe1212bc037"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7da13bb6fbadfafb474e0226a30570a3445cfd47c86296f2446dafbd77079ace"}, + {file = "lxml-6.1.0.tar.gz", hash = "sha256:bfd57d8008c4965709a919c3e9a98f76c2c7cb319086b3d26858250620023b13"}, ] [package.extras] @@ -3710,14 +3704,14 @@ types-webencodings = "*" [[package]] name = "types-jsonschema" -version = "4.26.0.20260202" +version = "4.26.0.20260508" description = "Typing stubs for jsonschema" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "types_jsonschema-4.26.0.20260202-py3-none-any.whl", hash = "sha256:41c95343abc4de9264e333a55e95dfb4d401e463856d0164eec9cb182e8746da"}, - {file = "types_jsonschema-4.26.0.20260202.tar.gz", hash = "sha256:29831baa4308865a9aec547a61797a06fc152b0dac8dddd531e002f32265cb07"}, + {file = "types_jsonschema-4.26.0.20260508-py3-none-any.whl", hash = "sha256:4ec1dea0a757c8c2e2aa7bc085612fb54e1ae9562428d5da6f26dd7a0f24dbc2"}, + {file = "types_jsonschema-4.26.0.20260508.tar.gz", hash = "sha256:ae0be85ac6ec0cb94a98f75f876b0620cf2afa3e37fdf8460203f4d05f745acb"}, ] [package.dependencies] @@ -3879,14 +3873,14 @@ files = [ [[package]] name = "urllib3" -version = "2.6.3" +version = "2.7.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4"}, - {file = "urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed"}, + {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, + {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, ] [package.extras] @@ -4050,4 +4044,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "862c41754b254346f8cba03162cdc392565d69fcd552cb08c1d39f11b860219a" +content-hash = "536527ee7ce227215b643b75479efabb75d4d7518626930f3dbc9768cc0927ce" diff --git a/pyproject.toml b/pyproject.toml index 7d3141b2d6..ccd8a2aed6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.153.0" +version = "1.154.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ @@ -66,7 +66,7 @@ dependencies = [ "prometheus-client>=0.6.0", # we use `order`, which arrived in attrs 19.2.0. # Note: 21.1.0 broke `/sync`, see https://github.com/matrix-org/synapse/issues/9936 - "attrs>=26.1.0,!=21.1.0", + "attrs>=19.2.0,!=21.1.0", "netaddr>=0.7.18", # Jinja 2.x is incompatible with MarkupSafe>=2.1. To ensure that admins do not # end up with a broken installation, with recent MarkupSafe but old Jinja, we diff --git a/rust/src/events/json_object.rs b/rust/src/events/json_object.rs new file mode 100644 index 0000000000..0ab54e8dc5 --- /dev/null +++ b/rust/src/events/json_object.rs @@ -0,0 +1,488 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::{collections::BTreeMap, sync::Arc}; + +use pyo3::{ + exceptions::{PyKeyError, PyTypeError}, + pyclass, pymethods, + types::{ + PyAnyMethods, PyIterator, PyList, PyListMethods, PyMapping, PySet, PySetMethods, PyTuple, + }, + Bound, IntoPyObject, IntoPyObjectExt, Py, PyAny, PyResult, Python, +}; +use pythonize::{depythonize, pythonize}; +use serde::{Deserialize, Serialize}; + +/// A generic class for representing immutable JSON objects. +/// +/// This is used for representing the `content` field of an event. +/// +/// The basic architecture here is to optimize for two things: +/// 1. Fast access of top-level keys (e.g. `event.content["key"]`) +/// 2. Pure Rust implementation. +#[derive(Serialize, Deserialize, Clone, Default)] +#[pyclass(mapping, frozen, skip_from_py_object)] +#[serde(transparent)] +pub struct JsonObject { + object: Arc, serde_json::Value>>, +} + +#[pymethods] +impl JsonObject { + #[new] + #[pyo3(signature = (content = None))] + fn new<'a, 'py>(content: Option<&'a Bound<'py, PyAny>>) -> PyResult { + let Some(content) = content else { + // If no content is provided, default to an empty object. + return Ok(Self::default()); + }; + + if let Ok(content) = content.cast::() { + // If the content is already a JsonObject, we can just clone the + // underlying map (this is safe as the object is immutable). + return Ok(JsonObject { + object: content.get().object.clone(), + }); + } + + let Ok(content) = content.cast::() else { + return Err(PyTypeError::new_err("'content' must be a mapping")); + }; + + // Use pythonize to try and convert from a mapping. + let content = depythonize(content)?; + Ok(Self { + object: Arc::new(content), + }) + } + + fn __len__(&self) -> usize { + self.object.len() + } + + fn __contains__(&self, key: &Bound<'_, PyAny>) -> bool { + // Match dict semantics: a non-string key is simply "not in" the + // mapping, rather than raising TypeError. + let Ok(key_str) = key.extract::<&str>() else { + return false; + }; + self.object.contains_key(key_str) + } + + fn __getitem__<'py>( + &self, + py: Python<'py>, + key: Bound<'_, PyAny>, + ) -> PyResult> { + // We only ever store string keys, so any non-string lookup is a miss. + // Raise KeyError (not TypeError) to match dict's behaviour. + let Ok(key_str) = key.extract::<&str>() else { + return Err(PyKeyError::new_err(key.unbind())); + }; + let Some(value) = self.object.get(key_str) else { + return Err(PyKeyError::new_err(key.unbind())); + }; + Ok(pythonize(py, value)?) + } + + fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + // The easiest way to get an iterator over the keys is to create a + // temporary list and call `iter()` on it. This is not the most + // efficient approach, but is much less boilerplate than implementing a + // custom iterator type. Since the keys are typically small in number + // this should be fine in practice. + let list = PyList::new(py, self.object.keys().map(Box::as_ref))?; + PyIterator::from_object(&list) + } + + // The view classes below each hold a `JsonObject` clone. This is cheap + // because the underlying map is behind an `Arc`, and lets the view outlive + // the originating object (matching dict_keys/values/items semantics in + // Python, which also keep the dict alive). + + fn keys(&self) -> JsonObjectKeysView { + JsonObjectKeysView { + object: self.clone(), + } + } + + fn values(&self) -> JsonObjectValuesView { + JsonObjectValuesView { + object: self.clone(), + } + } + + fn items(&self) -> JsonObjectItemsView { + JsonObjectItemsView { + object: self.clone(), + } + } + + #[pyo3(signature = (key, default=None))] + fn get<'py>( + &self, + py: Python<'py>, + key: Bound<'_, PyAny>, + default: Option>, + ) -> PyResult> { + // Non-string keys can never match, so treat them as a miss and return + // the caller-supplied default rather than raising. + let Ok(key_str) = key.extract::<&str>() else { + return Ok(default.into_pyobject(py)?); + }; + match self.object.get(key_str) { + Some(value) => Ok(pythonize(py, value)?), + None => Ok(default.into_pyobject(py)?), + } + } + + fn __eq__(&self, other: Bound<'_, PyAny>) -> bool { + // We support equality against any Python mapping (e.g. plain dicts), + // so callers can swap a JsonObject in without rewriting comparisons. + let Ok(mapping) = other.cast::() else { + return false; + }; + + let Ok(other_len) = mapping.len() else { + return false; + }; + + if other_len != self.object.len() { + return false; + } + + // We know the "other" is a mapping with the same number of fields as + // us. So we can convert it into a JsonObject and compare the underlying + // maps. + let Ok(other_dict) = depythonize(&other) else { + return false; + }; + + *self.object == other_dict + } + + // Since we implement comparisons with other types, we need to disable + // hashing to avoid violating the invariant that equal objects must have the + // same hash. + // + // Alternatively, we could only allow comparisons with other JsonObjects and + // allow hashing, but a) its nicer to be able to compare with arbitrary + // mappings and b) we don't really need hashing for these objects. + #[classattr] + const __hash__: Option> = None; + + fn __str__(&self) -> String { + serde_json::to_string(&self.object).expect("Value should be serializable") + } + + fn __repr__(&self) -> String { + format!("JsonObject({})", self.__str__()) + } +} + +/// Helper class returned by `JsonObject.keys()` to act as a view into the keys +/// of the object. +/// +/// This needs to both be iterable *and* operate like a set. +#[pyclass(frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct JsonObjectKeysView { + object: JsonObject, +} + +#[pymethods] +impl JsonObjectKeysView { + fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + // Create the iterator by making a temporary python list of the keys and + // calling `iter()` on it. + let list = PyList::new(py, self.object.object.keys().map(Box::as_ref))?; + PyIterator::from_object(&list) + } + + fn __len__(&self) -> usize { + self.object.__len__() + } + + fn __contains__(&self, key: &Bound<'_, PyAny>) -> bool { + self.object.__contains__(key) + } + + fn __eq__(&self, other: Bound<'_, PyAny>) -> bool { + let other_len = match other.len() { + Ok(len) => len, + Err(_) => return false, + }; + + if self.object.__len__() != other_len { + return false; + } + + for key in self.object.object.keys() { + if !matches!(other.contains(key.as_ref()), Ok(true)) { + return false; + } + } + + true + } + + // The set operators below match the behaviour of `dict.keys()` in Python: + // they accept any object that supports `__contains__` (for `&`) or is + // iterable (for `|`, `-`, `^`), not just sets. Each returns a fresh + // `PySet` so the caller gets a normal mutable Python set back. + // + // The `__r*__` variants are reflected operators, called by Python when + // the left-hand operand doesn't know how to combine with us. Since these + // operations are commutative for sets (or symmetric in the case of `^`), + // they just delegate. The asymmetric ops (`-`) need a separate impl. + + fn __and__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + // Iterate our (typically small) key set and probe `other`, which may + // be any container supporting `__contains__`. + let mut result = Vec::new(); + + for key in self.object.object.keys() { + if matches!(other.contains(key.as_ref()), Ok(true)) { + result.push(key.as_ref()); + } + } + + PySet::new(py, &result) + } + + fn __rand__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + self.__and__(py, other) + } + + fn __or__<'py>(&self, py: Python<'py>, other: Bound<'_, PyAny>) -> PyResult> { + // Union needs to enumerate both sides, so the right operand must be + // iterable (a bare `__contains__` is not enough). + let Ok(other_iter) = other.try_iter() else { + return Err(PyTypeError::new_err("Right operand must be iterable")); + }; + + let result = PySet::new(py, self.object.object.keys().map(Box::as_ref))?; + + // PySet handles dedup, so we can blindly add every element from the + // other iterable. + for item in other_iter { + let item = item?; + result.add(item)?; + } + + Ok(result) + } + + fn __ror__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + self.__or__(py, other) + } + + fn __sub__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + // `self - other`: keep our keys that are not in `other`. Only `other` + // needs to support `__contains__` here. + let mut result = Vec::new(); + + for key in self.object.object.keys() { + if matches!(other.contains(key.as_ref()), Ok(true)) { + continue; + } + result.push(key.as_ref()); + } + + PySet::new(py, &result) + } + + fn __rsub__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + // `other - self`: we need to enumerate `other`, so it must be + // iterable. Not symmetric with `__sub__`, hence a separate impl. + let Ok(other_iter) = other.try_iter() else { + return Err(PyTypeError::new_err("Left operand must be iterable")); + }; + + let result = PySet::empty(py)?; + + for item in other_iter { + let item = item?; + if self.object.__contains__(&item) { + continue; + } + result.add(item)?; + } + + Ok(result) + } + + fn __xor__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + // Symmetric difference: elements in exactly one side. Implemented as + // two filtered passes — one over our keys, one over `other`. + let Ok(other_iter) = other.try_iter() else { + return Err(PyTypeError::new_err("Right operand must be iterable")); + }; + + let result = PySet::empty(py)?; + + for key in self.object.object.keys() { + if matches!(other.contains(key.as_ref()), Ok(true)) { + continue; + } + result.add(key.as_ref())?; + } + + for item in other_iter { + let item = item?; + if self.object.__contains__(&item) { + continue; + } + result.add(item)?; + } + + Ok(result) + } + + fn __rxor__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + self.__xor__(py, other) + } + + fn isdisjoint(&self, other: Bound<'_, PyAny>) -> bool { + for key in self.object.object.keys() { + if matches!(other.contains(key.as_ref()), Ok(true)) { + return false; + } + } + + true + } +} + +/// Helper class returned by `JsonObject.values()` to act as a view into the +/// values of the object. +#[pyclass(frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct JsonObjectValuesView { + object: JsonObject, +} + +#[pymethods] +impl JsonObjectValuesView { + fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + // Create the iterator by making a temporary python list of the keys and + // calling `iter()` on it. + let list = PyList::empty(py); + for v in self.object.object.values() { + let py_value = pythonize(py, v)?.into_bound_py_any(py)?; + list.append(py_value)?; + } + + PyIterator::from_object(&list) + } + + fn __len__(&self) -> usize { + self.object.__len__() + } + + fn __contains__(&self, other: Bound<'_, PyAny>) -> bool { + // We compare by JSON equality rather than Python identity: convert + // the candidate into a `serde_json::Value` once and scan our values. + // Anything that fails to depythonize cannot match by definition. + let other_value: serde_json::Value = match depythonize(&other) { + Ok(v) => v, + Err(_) => return false, + }; + self.object.object.values().any(|v| *v == other_value) + } +} + +/// Helper class returned by `JsonObject.items()` to act as a view into the +/// items of the object. +/// +/// Technically this should be a set-like view according to Python semantics, +/// unless the values are unhashable. Since the values are immutable we could +/// support it, but it's more work and nobody seems to actually use the set +/// operations on `dict_items` in practice. +#[pyclass(frozen, skip_from_py_object)] +#[derive(Clone)] +pub struct JsonObjectItemsView { + object: JsonObject, +} + +#[pymethods] +impl JsonObjectItemsView { + fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + // Create the iterator by making a temporary python list of the keys and + // calling `iter()` on it. + let list = PyList::empty(py); + for (k, v) in self.object.object.iter() { + let py_key = k.as_ref().into_bound_py_any(py)?; + let py_value = pythonize(py, v)?.into_bound_py_any(py)?; + let item = PyTuple::new(py, [py_key, py_value])?; + list.append(item)?; + } + + PyIterator::from_object(&list) + } + + fn __len__(&self) -> usize { + self.object.__len__() + } + + fn __contains__(&self, other: Bound<'_, PyAny>) -> bool { + // `(key, value) in items` — only a 2-tuple can possibly match. We + // look the key up directly (avoiding a full scan) and then compare + // the stored value against `value` using JSON equality. + let Ok((key, value)) = other.extract::<(Bound<'_, PyAny>, Bound<'_, PyAny>)>() else { + return false; + }; + let Ok(key_str) = key.extract::<&str>() else { + return false; + }; + let Some(stored) = self.object.object.get(key_str) else { + return false; + }; + let other_value: serde_json::Value = match depythonize(&value) { + Ok(v) => v, + Err(_) => return false, + }; + *stored == other_value + } +} diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index 5f505abb91..e60cdb7078 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -21,21 +21,31 @@ //! Classes for representing Events. use pyo3::{ - types::{PyAnyMethods, PyModule, PyModuleMethods}, + types::{PyAnyMethods, PyMapping, PyModule, PyModuleMethods}, wrap_pyfunction, Bound, PyResult, Python, }; pub mod filter; mod internal_metadata; +mod json_object; pub mod signatures; pub mod unsigned; +use json_object::JsonObject; + /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + // Register the `JsonObject` class as a `Mapping` so that `isinstance` works. + PyMapping::register::(py)?; + let child_module = PyModule::new(py, "events")?; child_module.add_class::()?; child_module.add_class::()?; child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; child_module.add_function(wrap_pyfunction!(filter::event_visible_to_server_py, m)?)?; m.add_submodule(&child_module)?; diff --git a/rust/src/room_versions.rs b/rust/src/room_versions.rs index 47473cf200..2860bfdbd7 100644 --- a/rust/src/room_versions.rs +++ b/rust/src/room_versions.rs @@ -14,7 +14,11 @@ //! Rust implementation of room version definitions. -use std::sync::{Arc, LazyLock, RwLock}; +use std::{ + fmt::Display, + str::FromStr, + sync::{Arc, LazyLock, RwLock}, +}; use pyo3::{ exceptions::{PyKeyError, PyRuntimeError}, @@ -30,47 +34,47 @@ use pyo3::{ /// versions that they were used or introduced in. /// The concept of an 'event format version' is specific to Synapse (the /// specification does not mention this term.) -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] pub struct EventFormatVersions {} #[pymethods] impl EventFormatVersions { /// $id:server event id format: used for room v1 and v2 #[classattr] - const ROOM_V1_V2: i32 = 1; + pub const ROOM_V1_V2: i32 = 1; /// MSC1659-style $hash event id format: used for room v3 #[classattr] - const ROOM_V3: i32 = 2; + pub const ROOM_V3: i32 = 2; /// MSC1884-style $hash format: introduced for room v4 #[classattr] - const ROOM_V4_PLUS: i32 = 3; + pub const ROOM_V4_PLUS: i32 = 3; /// MSC4291 room IDs as hashes: introduced for room HydraV11 #[classattr] - const ROOM_V11_HYDRA_PLUS: i32 = 4; + pub const ROOM_V11_HYDRA_PLUS: i32 = 4; /// MSC4242 state DAGs: adds prev_state_events, removes auth_events #[classattr] - const ROOM_VMSC4242: i32 = 5; + pub const ROOM_VMSC4242: i32 = 5; } /// Enum to identify the state resolution algorithms. -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] pub struct StateResolutionVersions {} #[pymethods] impl StateResolutionVersions { /// Room v1 state res #[classattr] - const V1: i32 = 1; + pub const V1: i32 = 1; /// MSC1442 state res: room v2 and later #[classattr] - const V2: i32 = 2; + pub const V2: i32 = 2; /// MSC4297 state res #[classattr] - const V2_1: i32 = 3; + pub const V2_1: i32 = 3; } /// Room disposition constants. -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] pub struct RoomDisposition {} #[pymethods] @@ -94,18 +98,18 @@ impl PushRuleRoomFlag { /// An object which describes the unique attributes of a room version. #[pyclass(frozen, eq, hash, get_all, skip_from_py_object)] -#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct RoomVersion { /// The identifier for this version. pub identifier: &'static str, - /// One of the RoomDisposition constants. + /// One of the [`RoomDisposition`] constants. pub disposition: &'static str, - /// One of the EventFormatVersions constants. + /// One of the [`EventFormatVersions`] constants. pub event_format: i32, - /// One of the StateResolutionVersions constants. + /// One of the [`StateResolutionVersions`] constants. pub state_res: i32, pub enforce_key_validity: bool, - /// Before MSC2432, m.room.aliases had special auth rules and redaction rules. + /// Before MSC2432, `m.room.aliases` had special auth rules and redaction rules. pub special_case_aliases_auth: bool, /// Strictly enforce canonicaljson, do not allow: /// * Integers outside the range of [-2^53 + 1, 2^53 - 1] @@ -159,459 +163,204 @@ pub struct RoomVersion { pub msc4242_state_dags: bool, } -const ROOM_VERSION_V1: RoomVersion = RoomVersion { - identifier: "1", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V1_V2, - state_res: StateResolutionVersions::V1, - enforce_key_validity: false, - special_case_aliases_auth: true, - strict_canonicaljson: false, - limit_notifications_power_levels: false, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V2: RoomVersion = RoomVersion { - identifier: "2", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V1_V2, - state_res: StateResolutionVersions::V2, - enforce_key_validity: false, - special_case_aliases_auth: true, - strict_canonicaljson: false, - limit_notifications_power_levels: false, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V3: RoomVersion = RoomVersion { - identifier: "3", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V3, - state_res: StateResolutionVersions::V2, - enforce_key_validity: false, - special_case_aliases_auth: true, - strict_canonicaljson: false, - limit_notifications_power_levels: false, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V4: RoomVersion = RoomVersion { - identifier: "4", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: false, - special_case_aliases_auth: true, - strict_canonicaljson: false, - limit_notifications_power_levels: false, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V5: RoomVersion = RoomVersion { - identifier: "5", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: true, - strict_canonicaljson: false, - limit_notifications_power_levels: false, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V6: RoomVersion = RoomVersion { - identifier: "6", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V7: RoomVersion = RoomVersion { - identifier: "7", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V8: RoomVersion = RoomVersion { - identifier: "8", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: false, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V9: RoomVersion = RoomVersion { - identifier: "9", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V10: RoomVersion = RoomVersion { - identifier: "10", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -/// MSC3389 (Redaction changes for events with a relation) based on room version "10". -const ROOM_VERSION_MSC3389V10: RoomVersion = RoomVersion { - identifier: "org.matrix.msc3389.10", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: true, // Changed from v10 - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: true, - msc4242_state_dags: false, -}; - -/// MSC1767 (Extensible Events) based on room version "10". -const ROOM_VERSION_MSC1767V10: RoomVersion = RoomVersion { - identifier: "org.matrix.msc1767.10", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[PushRuleRoomFlag::EXTENSIBLE_EVENTS], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; +impl RoomVersion { + pub const V1: RoomVersion = RoomVersion { + identifier: "1", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V1_V2, + state_res: StateResolutionVersions::V1, + enforce_key_validity: false, + special_case_aliases_auth: true, + strict_canonicaljson: false, + limit_notifications_power_levels: false, + implicit_room_creator: false, + updated_redaction_rules: false, + restricted_join_rule: false, + restricted_join_rule_fix: false, + knock_join_rule: false, + msc3389_relation_redactions: false, + knock_restricted_join_rule: false, + enforce_int_power_levels: false, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, + }; + + pub const V2: RoomVersion = RoomVersion { + identifier: "2", + state_res: StateResolutionVersions::V2, + ..Self::V1 + }; + + pub const V3: RoomVersion = RoomVersion { + identifier: "3", + event_format: EventFormatVersions::ROOM_V3, + ..Self::V2 + }; + + pub const V4: RoomVersion = RoomVersion { + identifier: "4", + event_format: EventFormatVersions::ROOM_V4_PLUS, + ..Self::V3 + }; + + pub const V5: RoomVersion = RoomVersion { + identifier: "5", + enforce_key_validity: true, + ..Self::V4 + }; + + pub const V6: RoomVersion = RoomVersion { + identifier: "6", + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + ..Self::V5 + }; + + pub const V7: RoomVersion = RoomVersion { + identifier: "7", + knock_join_rule: true, + ..Self::V6 + }; + + pub const V8: RoomVersion = RoomVersion { + identifier: "8", + restricted_join_rule: true, + ..Self::V7 + }; + + pub const V9: RoomVersion = RoomVersion { + identifier: "9", + restricted_join_rule_fix: true, + ..Self::V8 + }; + + pub const V10: RoomVersion = RoomVersion { + identifier: "10", + knock_restricted_join_rule: true, + enforce_int_power_levels: true, + ..Self::V9 + }; + + /// MSC3389 (Redaction changes for events with a relation) based on room version "10". + pub const MSC3389V10: RoomVersion = RoomVersion { + identifier: "org.matrix.msc3389.10", + disposition: RoomDisposition::UNSTABLE, + msc3389_relation_redactions: true, + strict_event_byte_limits_room_versions: true, + ..Self::V10 + }; + + /// MSC1767 (Extensible Events) based on room version "10". + pub const MSC1767V10: RoomVersion = RoomVersion { + identifier: "org.matrix.msc1767.10", + disposition: RoomDisposition::UNSTABLE, + msc3931_push_features: &[PushRuleRoomFlag::EXTENSIBLE_EVENTS], + ..Self::V10 + }; + + /// MSC3757 (Restricting who can overwrite a state event) based on room version "10". + pub const MSC3757V10: RoomVersion = RoomVersion { + identifier: "org.matrix.msc3757.10", + disposition: RoomDisposition::UNSTABLE, + msc3757_enabled: true, + ..Self::V10 + }; + + pub const V11: RoomVersion = RoomVersion { + identifier: "11", + implicit_room_creator: true, // Used by MSC3820 + updated_redaction_rules: true, // Used by MSC3820 + strict_event_byte_limits_room_versions: true, + ..Self::V10 + }; + + /// MSC3757 (Restricting who can overwrite a state event) based on room version "11". + pub const MSC3757V11: RoomVersion = RoomVersion { + identifier: "org.matrix.msc3757.11", + disposition: RoomDisposition::UNSTABLE, + msc3757_enabled: true, + ..Self::V11 + }; + + pub const HYDRA_V11: RoomVersion = RoomVersion { + identifier: "org.matrix.hydra.11", + disposition: RoomDisposition::UNSTABLE, + event_format: EventFormatVersions::ROOM_V11_HYDRA_PLUS, + state_res: StateResolutionVersions::V2_1, + msc4289_creator_power_enabled: true, + msc4291_room_ids_as_hashes: true, + ..Self::V11 + }; + + pub const V12: RoomVersion = RoomVersion { + identifier: "12", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V11_HYDRA_PLUS, + state_res: StateResolutionVersions::V2_1, + msc4289_creator_power_enabled: true, + msc4291_room_ids_as_hashes: true, + ..Self::V11 + }; + + pub const MSC4242V12: RoomVersion = RoomVersion { + identifier: "org.matrix.msc4242.12", + disposition: RoomDisposition::UNSTABLE, + event_format: EventFormatVersions::ROOM_VMSC4242, + msc4242_state_dags: true, + ..Self::V12 + }; +} -/// MSC3757 (Restricting who can overwrite a state event) based on room version "10". -const ROOM_VERSION_MSC3757V10: RoomVersion = RoomVersion { - identifier: "org.matrix.msc3757.10", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: true, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; +impl Display for RoomVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.identifier.fmt(f) + } +} -const ROOM_VERSION_V11: RoomVersion = RoomVersion { - identifier: "11", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: true, // Used by MSC3820 - updated_redaction_rules: true, // Used by MSC3820 - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: true, // Changed from v10 - msc4242_state_dags: false, -}; +impl FromStr for &'static RoomVersion { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "1" => Ok(&RoomVersion::V1), + "2" => Ok(&RoomVersion::V2), + "3" => Ok(&RoomVersion::V3), + "4" => Ok(&RoomVersion::V4), + "5" => Ok(&RoomVersion::V5), + "6" => Ok(&RoomVersion::V6), + "7" => Ok(&RoomVersion::V7), + "8" => Ok(&RoomVersion::V8), + "9" => Ok(&RoomVersion::V9), + "10" => Ok(&RoomVersion::V10), + "11" => Ok(&RoomVersion::V11), + "12" => Ok(&RoomVersion::V12), + "org.matrix.msc1767.10" => Ok(&RoomVersion::MSC1767V10), + "org.matrix.msc3389.10" => Ok(&RoomVersion::MSC3389V10), + "org.matrix.msc3757.10" => Ok(&RoomVersion::MSC3757V10), + "org.matrix.msc3757.11" => Ok(&RoomVersion::MSC3757V11), + "org.matrix.hydra.11" => Ok(&RoomVersion::HYDRA_V11), + "org.matrix.msc4242.12" => Ok(&RoomVersion::MSC4242V12), + _ => Err(anyhow::anyhow!("Unknown room version: {}", s)), + } + } +} -/// MSC3757 (Restricting who can overwrite a state event) based on room version "11". -const ROOM_VERSION_MSC3757V11: RoomVersion = RoomVersion { - identifier: "org.matrix.msc3757.11", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: true, // Used by MSC3820 - updated_redaction_rules: true, // Used by MSC3820 - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: true, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: true, - msc4242_state_dags: false, -}; +impl<'py> IntoPyObject<'py> for &RoomVersion { + type Target = RoomVersion; -const ROOM_VERSION_HYDRA_V11: RoomVersion = RoomVersion { - identifier: "org.matrix.hydra.11", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_V11_HYDRA_PLUS, - state_res: StateResolutionVersions::V2_1, // Changed from v11 - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: true, // Used by MSC3820 - updated_redaction_rules: true, // Used by MSC3820 - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: true, // Changed from v11 - msc4291_room_ids_as_hashes: true, // Changed from v11 - strict_event_byte_limits_room_versions: true, - msc4242_state_dags: false, -}; + type Output = Bound<'py, RoomVersion>; -const ROOM_VERSION_V12: RoomVersion = RoomVersion { - identifier: "12", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V11_HYDRA_PLUS, - state_res: StateResolutionVersions::V2_1, // Changed from v11 - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: true, // Used by MSC3820 - updated_redaction_rules: true, // Used by MSC3820 - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: true, // Changed from v11 - msc4291_room_ids_as_hashes: true, // Changed from v11 - strict_event_byte_limits_room_versions: true, - msc4242_state_dags: false, -}; + type Error = PyErr; -const ROOM_VERSION_MSC4242V12: RoomVersion = RoomVersion { - identifier: "org.matrix.msc4242.12", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_VMSC4242, - state_res: StateResolutionVersions::V2_1, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: true, - updated_redaction_rules: true, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: true, - msc4291_room_ids_as_hashes: true, - strict_event_byte_limits_room_versions: true, - msc4242_state_dags: true, -}; + fn into_pyobject(self, py: Python<'py>) -> Result { + self.clone().into_pyobject(py) + } +} /// Helper class for managing the known room versions, and providing dict-like /// access to them for Python. @@ -652,7 +401,7 @@ impl KnownRoomVersionsMapping { return Ok(()); } - versions.push(*version.get()); + versions.push(version.get().clone()); Ok(()) } @@ -666,7 +415,7 @@ impl KnownRoomVersionsMapping { versions .iter() .find(|v| v.identifier == key) - .copied() + .cloned() .ok_or_else(|| PyKeyError::new_err(key.to_string())) } @@ -698,7 +447,7 @@ impl KnownRoomVersionsMapping { // like a Set *and* has a stable ordering). We don't depend on this, so // for simplicity we just return a list of the items. let versions = self.versions.read().unwrap(); - Ok(versions.iter().map(|v| (v.identifier, *v)).collect()) + Ok(versions.iter().map(|v| (v.identifier, v.clone())).collect()) } #[pyo3(signature = (key, default=None))] @@ -715,8 +464,8 @@ impl KnownRoomVersionsMapping { }; let versions = self.versions.read().unwrap(); - if let Some(version) = versions.iter().find(|v| v.identifier == key).copied() { - return Ok(Some(version.into_bound_py_any(py)?)); + if let Some(version) = versions.iter().find(|v| v.identifier == key) { + return Ok(Some(version.clone().into_bound_py_any(py)?)); } Ok(default) @@ -757,21 +506,21 @@ impl<'py> IntoPyObject<'py> for &KnownRoomVersionsMapping { /// support all experimental room versions. static KNOWN_ROOM_VERSIONS: LazyLock = LazyLock::new(|| { let vec = vec![ - ROOM_VERSION_V1, - ROOM_VERSION_V2, - ROOM_VERSION_V3, - ROOM_VERSION_V4, - ROOM_VERSION_V5, - ROOM_VERSION_V6, - ROOM_VERSION_V7, - ROOM_VERSION_V8, - ROOM_VERSION_V9, - ROOM_VERSION_V10, - ROOM_VERSION_V11, - ROOM_VERSION_V12, - ROOM_VERSION_MSC3757V10, - ROOM_VERSION_MSC3757V11, - ROOM_VERSION_HYDRA_V11, + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + RoomVersion::V11, + RoomVersion::V12, + RoomVersion::MSC3757V10, + RoomVersion::MSC3757V11, + RoomVersion::HYDRA_V11, ]; KnownRoomVersionsMapping { @@ -783,82 +532,82 @@ static KNOWN_ROOM_VERSIONS: LazyLock = LazyLock::new(| /// /// This should contain all room versions that we know about. #[pyclass(frozen)] -pub struct RoomVersions {} +struct RoomVersions {} #[pymethods] #[allow(non_snake_case)] impl RoomVersions { #[classattr] fn V1(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V1.into_py_any(py) + RoomVersion::V1.into_py_any(py) } #[classattr] fn V2(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V2.into_py_any(py) + RoomVersion::V2.into_py_any(py) } #[classattr] fn V3(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V3.into_py_any(py) + RoomVersion::V3.into_py_any(py) } #[classattr] fn V4(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V4.into_py_any(py) + RoomVersion::V4.into_py_any(py) } #[classattr] fn V5(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V5.into_py_any(py) + RoomVersion::V5.into_py_any(py) } #[classattr] fn V6(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V6.into_py_any(py) + RoomVersion::V6.into_py_any(py) } #[classattr] fn V7(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V7.into_py_any(py) + RoomVersion::V7.into_py_any(py) } #[classattr] fn V8(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V8.into_py_any(py) + RoomVersion::V8.into_py_any(py) } #[classattr] fn V9(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V9.into_py_any(py) + RoomVersion::V9.into_py_any(py) } #[classattr] fn V10(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V10.into_py_any(py) + RoomVersion::V10.into_py_any(py) } #[classattr] fn MSC1767v10(py: Python<'_>) -> PyResult> { - ROOM_VERSION_MSC1767V10.into_py_any(py) + RoomVersion::MSC1767V10.into_py_any(py) } #[classattr] fn MSC3389v10(py: Python<'_>) -> PyResult> { - ROOM_VERSION_MSC3389V10.into_py_any(py) + RoomVersion::MSC3389V10.into_py_any(py) } #[classattr] fn MSC3757v10(py: Python<'_>) -> PyResult> { - ROOM_VERSION_MSC3757V10.into_py_any(py) + RoomVersion::MSC3757V10.into_py_any(py) } #[classattr] fn V11(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V11.into_py_any(py) + RoomVersion::V11.into_py_any(py) } #[classattr] fn MSC3757v11(py: Python<'_>) -> PyResult> { - ROOM_VERSION_MSC3757V11.into_py_any(py) + RoomVersion::MSC3757V11.into_py_any(py) } #[classattr] fn HydraV11(py: Python<'_>) -> PyResult> { - ROOM_VERSION_HYDRA_V11.into_py_any(py) + RoomVersion::HYDRA_V11.into_py_any(py) } #[classattr] fn V12(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V12.into_py_any(py) + RoomVersion::V12.into_py_any(py) } #[classattr] fn MSC4242v12(py: Python<'_>) -> PyResult> { - ROOM_VERSION_MSC4242V12.into_py_any(py) + RoomVersion::MSC4242V12.into_py_any(py) } } diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 9484e017d1..037527031a 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://famedly.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://famedly.github.io/synapse/schema/synapse/v1.153/synapse-config.schema.json +$id: https://famedly.github.io/synapse/schema/synapse/v1.154/synapse-config.schema.json type: object properties: famedly_maximum_refresh_token_lifetime: @@ -4635,11 +4635,15 @@ properties: type: boolean description: >- Use this setting to keep a user's profile fields in sync with - information from the identity provider. Currently only syncing the - displayname is supported. Fields are checked on every SSO login, and - are updated if necessary. Note that enabling this option will override - user profile information, regardless of whether users have opted-out - of syncing that information when first signing in. + information from the identity provider. Fields are checked on every + SSO login, and are updated if necessary. Note that enabling this + option will override user profile information, regardless of whether + users have opted-out of syncing that information when first signing + in. + Fields that will be synced: + * displayname + * picture - only if Synapse media repository is running in the main + process (i.e. not workerized) and media is stored locally default: false examples: - client_whitelist: diff --git a/synapse/__init__.py b/synapse/__init__.py index 2bed060878..3acfc1a0d7 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -65,7 +65,8 @@ except ImportError: pass -# Teach canonicaljson how to serialise immutabledicts. + +# Teach canonicaljson how to serialise immutabledicts and JsonObjects. try: from canonicaljson import register_preserialisation_callback from immutabledict import immutabledict @@ -79,6 +80,12 @@ def _immutabledict_cb(d: immutabledict) -> dict[str, Any]: return dict(d) register_preserialisation_callback(immutabledict, _immutabledict_cb) + + # Teach canonicaljson how to serialise JsonObjects, which is just to + # convert them to dicts. + from synapse.synapse_rust.events import JsonObject # noqa: E402 + + register_preserialisation_callback(JsonObject, lambda obj: dict(obj)) except ImportError: pass diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index a1d4eb3ed6..76bf4f992f 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -616,3 +616,7 @@ def read_config( # Tracked in: https://github.com/element-hq/synapse/issues/19691 # Note that this is only applicable to legacy auth, not MAS integration (OAuth 2.0). self.msc4450_enabled: bool = experimental.get("msc4450_enabled", False) + + # MSC4455: Preview URL capability + # Tracked in: https://github.com/element-hq/synapse/issues/19719 + self.msc4452_enabled: bool = experimental.get("msc4452_enabled", False) diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 96e7a5f192..f84358f0a6 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -242,7 +242,8 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.thumbnail_requirements = parse_thumbnail_requirements( config.get("thumbnail_sizes", DEFAULT_THUMBNAIL_SIZES) ) - self.url_preview_enabled = config.get("url_preview_enabled", False) + self.url_preview_enabled = bool(config.get("url_preview_enabled", False)) + if self.url_preview_enabled: check_requirements("url-preview") diff --git a/synapse/event_auth.py b/synapse/event_auth.py index fd35da8ba0..d9750651fa 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -61,7 +61,8 @@ EventFormatVersions, RoomVersion, ) -from synapse.events import FrozenEventVMSC4242, is_creator +from synapse.events import is_creator +from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.state import CREATE_KEY from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import ( @@ -187,8 +188,8 @@ async def check_state_independent_auth_rules( return # State DAGs 2. Considering the event's prev_state_events: - if event.room_version.msc4242_state_dags: - prev_state_events_ids = set(cast(FrozenEventVMSC4242, event).prev_state_events) + if supports_msc4242_state_dag(event): + prev_state_events_ids = set(event.prev_state_events) # Fetch all of the `prev_state_events` prev_state_events = {} # Try to load the `prev_state_events` from `batched_auth_events` initially as @@ -515,8 +516,7 @@ def _check_create(event: "EventBase") -> None: raise AuthError(403, "Create event has prev events") # State DAGs 1.2 If it has any prev_state_events, reject. - if event.room_version.msc4242_state_dags: - assert isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event): if len(event.prev_state_events) > 0: raise AuthError(403, "Create event has prev state events") diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 487bed59bd..be32b5649f 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -44,9 +44,15 @@ StickyEvent, ) from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions -from synapse.synapse_rust.events import EventInternalMetadata, Signatures, Unsigned +from synapse.synapse_rust.events import ( + EventInternalMetadata, + JsonObject, + Signatures, + Unsigned, +) from synapse.types import ( JsonDict, + JsonMapping, StateKey, StrCollection, ) @@ -206,17 +212,29 @@ def __init__( ): assert room_version.event_format == self.format_version + if "content" in event_dict: + event_dict["content"] = JsonObject(event_dict["content"]) + + # We intern these strings because they turn up a lot (especially when + # caching). + event_dict = intern_dict(event_dict) + + if USE_FROZEN_DICTS: + frozen_dict = freeze(event_dict) + else: + frozen_dict = event_dict + self.room_version = room_version self.signatures = Signatures(signatures) self.unsigned = Unsigned(unsigned) self.rejected_reason = rejected_reason - self._dict = event_dict + self._dict = frozen_dict self.internal_metadata = EventInternalMetadata(internal_metadata_dict) depth: DictProperty[int] = DictProperty("depth") - content: DictProperty[JsonDict] = DictProperty("content") + content: DictProperty[JsonMapping] = DictProperty("content") hashes: DictProperty[dict[str, str]] = DictProperty("hashes") origin_server_ts: DictProperty[int] = DictProperty("origin_server_ts") sender: DictProperty[str] = DictProperty("sender") @@ -259,7 +277,14 @@ def get_state_key(self) -> str | None: def get_dict(self) -> JsonDict: """Convert the event to a dictionary suitable for serialisation.""" + d = dict(self._dict) + if "content" in d: + # Convert the content (which is a JsonObject) back to a dict. Json + # serialization should handle JsonObjects fine, but for sanities + # sake we want `get_dict()` and `get_pdu_json()` to return plain + # dicts. + d["content"] = dict(self.content) d.update( { "signatures": self.signatures.as_dict(), @@ -425,19 +450,10 @@ def __init__( unsigned = event_dict.pop("unsigned", {}) - # We intern these strings because they turn up a lot (especially when - # caching). - event_dict = intern_dict(event_dict) - - if USE_FROZEN_DICTS: - frozen_dict = freeze(event_dict) - else: - frozen_dict = event_dict - self._event_id = event_dict["event_id"] super().__init__( - frozen_dict, + event_dict, room_version=room_version, signatures=signatures, unsigned=unsigned, @@ -479,19 +495,10 @@ def __init__( unsigned = event_dict.pop("unsigned", {}) - # We intern these strings because they turn up a lot (especially when - # caching). - event_dict = intern_dict(event_dict) - - if USE_FROZEN_DICTS: - frozen_dict = freeze(event_dict) - else: - frozen_dict = event_dict - self._event_id: str | None = None super().__init__( - frozen_dict, + event_dict, room_version=room_version, signatures=signatures, unsigned=unsigned, @@ -617,7 +624,7 @@ class FrozenEventVMSC4242(FrozenEventV4): """FrozenEventVMSC4242, which differs from FrozenEventV4 only in the addition of prev_state_events""" format_version = EventFormatVersions.ROOM_VMSC4242 - prev_state_events: DictProperty[list[str]] = DictProperty("prev_state_events") + prev_state_events: DictProperty[StrCollection] = DictProperty("prev_state_events") def __init__( self, diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 78eb98e1e5..742bb82da9 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -132,7 +132,7 @@ async def build( prev_event_ids: list[str], auth_event_ids: list[str] | None, depth: int | None = None, - prev_state_events: list[str] | None = None, + prev_state_events: StrCollection | None = None, ) -> EventBase: """Transform into a fully signed and hashed event diff --git a/synapse/events/py_protocol.py b/synapse/events/py_protocol.py new file mode 100644 index 0000000000..d9ac8c066f --- /dev/null +++ b/synapse/events/py_protocol.py @@ -0,0 +1,88 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# +"""Type-narrowing helpers for `EventBase`. + +`EventBase` subclasses are split by room version (e.g. `FrozenEventV4`, +`FrozenEventVMSC4242`), and certain attributes — such as `prev_state_events` +on MSC4242 events — only exist on a subset of those subclasses. Branching on +`room_version.` at runtime tells us *which* subclass we have, but the +type checker can't see that link without an `isinstance` cast at every call +site. + +This module provides "marker" subclasses of `EventBase` (`MSC4242Event`, +etc.) paired with `TypeIs`-returning predicates (`supports_msc4242_state_dag`, +etc.). A single call to the predicate both performs the room-version check +and narrows the type — replacing the `if room_version.foo: assert +isinstance(event, FrozenEventV...)` idiom. + +The marker classes are *type-only*: their metaclass raises on `isinstance` +so they cannot be misused as real runtime classes. Add new markers and +predicates here when a new room-version feature gates access to additional +attributes. +""" + +import abc +from typing import ( + TYPE_CHECKING, + Sequence, +) + +from typing_extensions import TypeIs + +from synapse.events import EventBase + +if TYPE_CHECKING: + from synapse.events.snapshot import EventContext, EventPersistencePair + + +class _DisableIsInstance(abc.ABCMeta): + """Metaclass which disables isinstance checks on classes which use it, by + making isinstance() raise NotImplementedError. + + This is used to prevent isinstance checks on EventProtocol, which is a + helper class used for type narrowing of EventBase objects, but which should + not be used for isinstance checks itself (as its purely type annotation + rather than a real class). + """ + + def __instancecheck__(cls, instance: object) -> bool: + raise NotImplementedError("Instance cannot be used.") + + +class EventProtocol(EventBase, metaclass=_DisableIsInstance): + """Helper subclass that allows type narrowing for `EventBase` objects.""" + + +class MSC4242Event(EventProtocol): + """Marker protocol for events in MSC4242 rooms. This allows us to narrow the + type of events.""" + + prev_state_events: list[str] + + +def supports_msc4242_state_dag(event: EventBase) -> TypeIs[MSC4242Event]: + """Returns true if the given event is in a room that supports state DAGs + (MSC4242)""" + + return event.room_version.msc4242_state_dags + + +def all_supports_msc4242_state_dag( + obj: Sequence["EventPersistencePair"], +) -> TypeIs[Sequence[tuple[MSC4242Event, "EventContext"]]]: + """Returns true if the given sequence of events are all in a room that + supports state DAGs (MSC4242)""" + + return all(event.room_version.msc4242_state_dags for event, _ in obj) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 926c81b83d..adbede7f16 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -1032,7 +1032,7 @@ def strip_event(event: EventBase) -> JsonDict: return { "type": event.type, "state_key": event.state_key, - "content": event.content, + "content": dict(event.content), "sender": event.sender, } diff --git a/synapse/events/validator.py b/synapse/events/validator.py index ff22b2287f..e8d6cc9710 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -22,7 +22,6 @@ from typing import cast import jsonschema -from pydantic import Field, StrictBool, StrictStr from synapse.api.constants import ( MAX_ALIAS_LENGTH, @@ -40,10 +39,8 @@ CANONICALJSON_MIN_INT, validate_canonicaljson, ) -from synapse.http.servlet import validate_json_object from synapse.storage.controllers.state import server_acl_evaluator_from_event -from synapse.types import EventID, JsonDict, RoomID, StrCollection, UserID -from synapse.types.rest import RequestBodyModel +from synapse.types import EventID, JsonDict, JsonMapping, RoomID, StrCollection, UserID class EventValidator: @@ -116,29 +113,18 @@ def validate_new(self, event: EventBase, config: HomeServerConfig) -> None: cls=POWER_LEVELS_VALIDATOR, ) except jsonschema.ValidationError as e: - if e.path: - # example: "users_default": '0' is not of type 'integer' - # cast safety: path entries can be integers, if we fail to validate - # items in an array. However, the POWER_LEVELS_SCHEMA doesn't expect - # to see any arrays. - message = ( - '"' + cast(str, e.path[-1]) + '": ' + e.message # noqa: B306 - ) - # jsonschema.ValidationError.message is a valid attribute - else: - # example: '0' is not of type 'integer' - message = e.message # noqa: B306 - # jsonschema.ValidationError.message is a valid attribute - - raise SynapseError( - code=400, - msg=message, - errcode=Codes.BAD_JSON, - ) + raise _validation_error_to_api_error(e) # If the event contains a mentions key, validate it. if EventContentFields.MENTIONS in event.content: - validate_json_object(event.content[EventContentFields.MENTIONS], Mentions) + try: + jsonschema.validate( + instance=event.content[EventContentFields.MENTIONS], + schema=MENTIONS_SCHEMA, + cls=MENTIONS_VALIDATOR, + ) + except jsonschema.ValidationError as e: + raise _validation_error_to_api_error(e) def _validate_retention(self, event: EventBase) -> None: """Checks that an event that defines the retention policy for a room respects the @@ -245,7 +231,7 @@ def validate_builder(self, event: EventBase | EventBuilder) -> None: self._ensure_state_event(event) - def _ensure_strings(self, d: JsonDict, keys: StrCollection) -> None: + def _ensure_strings(self, d: JsonMapping, keys: StrCollection) -> None: for s in keys: if s not in d: raise SynapseError(400, "'%s' not in content" % (s,)) @@ -284,10 +270,16 @@ def _ensure_state_event(self, event: EventBase | EventBuilder) -> None: }, } - -class Mentions(RequestBodyModel): - user_ids: list[StrictStr] = Field(default_factory=list) - room: StrictBool = False +MENTIONS_SCHEMA = { + "type": "object", + "properties": { + "user_ids": { + "type": "array", + "items": {"type": "string"}, + }, + "room": {"type": "boolean"}, + }, +} # This could return something newer than Draft 7, but that's the current "latest" @@ -295,14 +287,45 @@ class Mentions(RequestBodyModel): def _create_validator(schema: JsonDict) -> type[jsonschema.Draft7Validator]: validator = jsonschema.validators.validator_for(schema) - # by default jsonschema does not consider a immutabledict to be an object so - # we need to use a custom type checker + # by default jsonschema does not consider a immutabledict to be an object, or + # a tuple to be an array (frozenutils freezes lists to tuples), so we need a + # custom type checker for both. # https://python-jsonschema.readthedocs.io/en/stable/validate/?highlight=object#validating-with-additional-types type_checker = validator.TYPE_CHECKER.redefine( "object", lambda checker, thing: isinstance(thing, collections.abc.Mapping) + ).redefine( + "array", + lambda checker, thing: isinstance(thing, collections.abc.Sequence), ) return jsonschema.validators.extend(validator, type_checker=type_checker) +def _validation_error_to_api_error(err: jsonschema.ValidationError) -> SynapseError: + """ + Converts a JSONSchema `ValidationError` to a `SynapseError` that can be thrown + to give a Matrix API-compatible 400 Bad Request response with `M_BAD_JSON` code + and a descriptive error message. + """ + if err.path: + # example: "users_default": '0' is not of type 'integer' + # cast safety: path entries can be integers, if we fail to validate + # items in an array. However, the POWER_LEVELS_SCHEMA doesn't expect + # to see any arrays. + message = '"' + cast(str, err.path[-1]) + '": ' + err.message + # jsonschema.ValidationError.message is a valid attribute + else: + # example: '0' is not of type 'integer' + message = err.message + # jsonschema.ValidationError.message is a valid attribute + + return SynapseError( + code=400, + msg=message, + errcode=Codes.BAD_JSON, + ) + + POWER_LEVELS_VALIDATOR = _create_validator(POWER_LEVELS_SCHEMA) + +MENTIONS_VALIDATOR = _create_validator(MENTIONS_SCHEMA) diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index 51a752472f..0b1251f9e3 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -32,7 +32,8 @@ from synapse.api.constants import Direction, EventTypes, Membership from synapse.api.errors import SynapseError -from synapse.events import EventBase, FrozenEventVMSC4242 +from synapse.events import EventBase +from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.events.utils import FilteredEvent from synapse.types import ( JsonMapping, @@ -495,8 +496,7 @@ async def _redact_all_events( try: prev_state_events = None - if room_version.msc4242_state_dags: - assert isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event): prev_state_events = event.prev_state_events assert prev_state_events is not None, ( "Parent event of redaction has no `prev_state_events` which should be impossible as `prev_state_events` is a required field in MSC4242 rooms" diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c3ff8d26cb..1761f169cc 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -273,11 +273,22 @@ async def _maybe_backfill_inner( ) ] - # we now have a list of potential places to backpaginate from. We prefer to - # start with the most recent (ie, max depth), so let's sort the list. + # we now have a list of potential places to backpaginate from. Figure out which + # ones we should prefer, so let's sort the list. sorted_backfill_points: list[_BackfillPoint] = sorted( backwards_extremities, - key=lambda e: -int(e.depth), + key=lambda e: ( + # Prefer backfill points that are closer to the `current_depth` + # (absolute distance) + abs(current_depth - e.depth), + # For the tie-break, we care about events that are actually in the past + # as they're more likely to reveal history that we can return (something + # absolutely in the past is better than something can potentially extend + # into the past). + # + # This sorts ascending so 0 sorts before 1 + 0 if current_depth >= e.depth else 1, + ), ) logger.debug( @@ -298,7 +309,7 @@ async def _maybe_backfill_inner( str(len(sorted_backfill_points)), ) - # If we have no backfill points lower than the `current_depth` then either we + # If we have no backfill points lower than the `nearby_depth` then either we # can a) bail or b) still attempt to backfill. We opt to try backfilling anyway # just in case we do get relevant events. This is good for eventual consistency # sake but we don't need to block the client for something that is just as diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index e314180e12..7cbc52d350 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -1166,9 +1166,10 @@ async def _compute_event_context_with_maybe_missing_prevs( return await self._state_handler.compute_event_context(event) logger.info( - "Event %s is missing prev_events %s: calculating state for a " - "backwards extremity", + "_compute_event_context_with_maybe_missing_prevs(event_id=%s): Event in room %s is missing prev_events %s: " + "calculating state for a backwards extremity", event_id, + room_id, shortstr(missing_prevs), ) # Calculate the state after each of the previous events, and @@ -1186,16 +1187,21 @@ async def _compute_event_context_with_maybe_missing_prevs( # Ask the remote server for the states we don't # know about - for p in missing_prevs: - logger.info("Requesting state after missing prev_event %s", p) + for missing_prev in missing_prevs: + logger.info( + "_compute_event_context_with_maybe_missing_prevs(event_id=%s): Requesting state from %s for missing prev_event %s", + event_id, + dest, + missing_prev, + ) - with nested_logging_context(p): + with nested_logging_context(missing_prev): # note that if any of the missing prevs share missing state or # auth events, the requests to fetch those events are deduped # by the get_pdu_cache in federation_client. remote_state_map = ( await self._get_state_ids_after_missing_prev_event( - dest, room_id, p + dest, room_id, missing_prev ) ) @@ -1225,7 +1231,11 @@ async def _compute_event_context_with_maybe_missing_prevs( except Exception as e: logger.warning( - "Error attempting to resolve state at missing prev_events: %s", e + "_compute_event_context_with_maybe_missing_prevs(event_id=%s): Error attempting to resolve state from " + "%s for missing prev_events: %s", + event_id, + dest, + e, ) raise FederationError( "ERROR", diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index a660b9f3f3..5a07b5e8c1 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -53,8 +53,9 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.api.urls import ConsentURIBuilder from synapse.event_auth import validate_event_for_room_version -from synapse.events import EventBase, FrozenEventVMSC4242, relation_from_event +from synapse.events import EventBase, relation_from_event from synapse.events.builder import EventBuilder +from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.events.snapshot import ( EventContext, EventPersistencePair, @@ -79,6 +80,7 @@ Requester, RoomAlias, StateMap, + StrCollection, StreamToken, UserID, create_requester, @@ -589,7 +591,7 @@ async def create_event( state_map: StateMap[str] | None = None, for_batch: bool = False, current_state_group: int | None = None, - prev_state_events: list[str] | None = None, + prev_state_events: StrCollection | None = None, delay_id: str | None = None, ) -> tuple[EventBase, UnpersistedEventContextBase]: """ @@ -982,7 +984,7 @@ async def create_and_send_nonmember_event( ignore_shadow_ban: bool = False, outlier: bool = False, depth: int | None = None, - prev_state_events: list[str] | None = None, + prev_state_events: StrCollection | None = None, delay_id: str | None = None, ) -> tuple[EventBase, int]: """ @@ -1127,7 +1129,7 @@ async def _create_and_send_nonmember_event_locked( ignore_shadow_ban: bool = False, outlier: bool = False, depth: int | None = None, - prev_state_events: list[str] | None = None, + prev_state_events: StrCollection | None = None, delay_id: str | None = None, ) -> tuple[EventBase, int]: room_id = event_dict["room_id"] @@ -1253,7 +1255,7 @@ async def create_new_client_event( state_map: StateMap[str] | None = None, for_batch: bool = False, current_state_group: int | None = None, - prev_state_events: list[str] | None = None, + prev_state_events: StrCollection | None = None, ) -> tuple[EventBase, UnpersistedEventContextBase]: """Create a new event for a local client. If bool for_batch is true, will create an event using the prev_event_ids, and will create an event context for @@ -1602,8 +1604,7 @@ async def handle_new_client_event( auth_event = event_id_to_event.get(event_id) if auth_event: batched_auth_events[event_id] = auth_event - if event.room_version.msc4242_state_dags: - assert isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event): # State DAG rooms will check that the prev_state_events are not rejected. # To do that, we need to make sure we pass in the prev_state_events as # batched_auth_events, else we will fail the event due to the @@ -1872,7 +1873,7 @@ async def cache_joined_hosts_for_events( state_entry = await self.state.resolve_state_groups_for_events( event.room_id, event_ids=event.prev_state_events - if isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event) else event.prev_event_ids(), ) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 5bfdb10f26..81a01e786f 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -719,12 +719,12 @@ async def clone_existing_room( spam_check = await self._spam_checker_module_callbacks.user_may_create_room( user_id, { - "creation_content": creation_content, + "creation_content": dict(creation_content), "initial_state": [ { "type": state_key[0], "state_key": state_key[1], - "content": event_content, + "content": dict(event_content), } for state_key, event_content in initial_state.items() ], @@ -1450,7 +1450,7 @@ async def _send_events_for_new_room( room_config: JsonDict, invite_list: list[str], initial_state: MutableStateMap, - creation_content: JsonDict, + creation_content: JsonMapping, room_alias: RoomAlias | None = None, power_level_content_override: JsonDict | None = None, creator_join_profile: JsonDict | None = None, @@ -1521,7 +1521,7 @@ async def _send_events_for_new_room( async def create_event( etype: str, - content: JsonDict, + content: JsonMapping, for_batch: bool, **kwargs: Any, ) -> tuple[EventBase, synapse.events.snapshot.UnpersistedEventContextBase]: @@ -1574,6 +1574,7 @@ async def create_event( if creation_event_with_context is None: # MSC2175 removes the creator field from the create event. if not room_version.implicit_room_creator: + creation_content = dict(creation_content) creation_content["creator"] = creator_id creation_event, unpersisted_creation_context = await create_event( EventTypes.Create, creation_content, False diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py index e46e6dc2ef..d19815d6e2 100644 --- a/synapse/handlers/room_policy.py +++ b/synapse/handlers/room_policy.py @@ -251,8 +251,8 @@ async def ask_policy_server_to_sign_event( # Note: if the policy server and event sender are the same server, the sender # might not have added policy server signatures to the event for whatever reason. # When this happens, we don't want to obliterate the event's existing signatures - # because the event will fail authorization. This is why we add defaults rather - # than simply `update` the signatures on the event. + # because the event will fail authorization. This is why we add items individually + # rather than simply `update` the signatures on the event. # # This situation can happen if the homeserver and policy server parts are # logically the same server, but run by different software. For example, Synapse @@ -261,7 +261,9 @@ async def ask_policy_server_to_sign_event( # servers need to manually fetch signatures for. This is the code that allows # those events to continue working (because they're legally sent, even if missing # the policy server signature). - event.signatures.update(signature) + signatures = signature.get(policy_server.server_name, {}) + for key_id, sig in signatures.items(): + event.signatures.add_signature(policy_server.server_name, key_id, sig) except HttpResponseException as ex: # re-wrap HTTP errors as `SynapseError` so they can be proxied to clients directly raise ex.to_synapse_error() from ex diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 1cc587d4a7..4d6287b147 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -150,6 +150,23 @@ async def wait_for_sync_for_user( # events or future events if the user is nefariously, manually modifying the # token. if from_token is not None: + # Work around a bug where older Synapse versions gave out tokens "from the + # future", i.e. that are ahead of the tokens persisted in the DB. This could + # also happen if a user is intentionally messing with the token so this also + # acts as sanitization/validation. + # + # If the token has positions ahead of our persisted positions in the + # database (invalid), then we simply use our max persisted position (recover + # gracefully); instead of waiting for a position that may never come around. + # + # FIXME: For Sliding Sync, instead of bounding the token, we should detect + # the invalid future position and raise a `M_UNKNOWN_POS` error. + from_token = SlidingSyncStreamToken( + stream_token=await self.event_sources.bound_future_token( + from_token.stream_token + ), + connection_position=from_token.connection_position, + ) # We need to make sure this worker has caught up with the token. If # this returns false, it means we timed out waiting, and we should # just return an empty response. @@ -855,20 +872,10 @@ async def get_room_sync_data( # For incremental syncs, we can do this first to determine if something relevant # has changed and strategically avoid fetching other costly things. room_state_delta_id_map: MutableStateMap[str] = {} - name_event_id: str | None = None membership_changed = False name_changed = False avatar_changed = False - if initial: - # Check whether the room has a name set - name_state_ids = await self.get_current_state_ids_at( - room_id=room_id, - room_membership_for_user_at_to_token=room_membership_for_user_at_to_token, - state_filter=StateFilter.from_types([(EventTypes.Name, "")]), - to_token=to_token, - ) - name_event_id = name_state_ids.get((EventTypes.Name, "")) - else: + if not initial: assert from_bound is not None # TODO: Limit the number of state events we're about to send down @@ -916,6 +923,27 @@ async def get_room_sync_data( ): avatar_changed = True + # If a room has an m.room.name event with an absent, null, or empty + # name field, it should be treated the same as a room with no + # m.room.name event. + # https://spec.matrix.org/v1.17/client-server-api/#mroomname + # + # TODO: Should we also check for `EventTypes.CanonicalAlias` + # (`m.room.canonical_alias`) as a fallback for the room name? see + # https://github.com/matrix-org/matrix-spec-proposals/pull/4186/changes#r2860107511 + room_name: str | None = None + if initial or name_changed: + # Check whether the room has a name set + name_states = await self.get_current_state_at( + room_id=room_id, + room_membership_for_user_at_to_token=room_membership_for_user_at_to_token, + state_filter=StateFilter.from_types([(EventTypes.Name, "")]), + to_token=to_token, + ) + name_event = name_states.get((EventTypes.Name, "")) + if name_event is not None: + room_name = name_event.content.get("name") + # We only need the room summary for calculating heroes, however if we do # fetch it then we can use it to calculate `joined_count` and # `invited_count`. @@ -932,12 +960,13 @@ async def get_room_sync_data( hero_user_ids: list[str] = [] # TODO: Should we also check for `EventTypes.CanonicalAlias` # (`m.room.canonical_alias`) as a fallback for the room name? see - # https://github.com/matrix-org/matrix-spec-proposals/pull/3575#discussion_r1671260153 + # https://github.com/matrix-org/matrix-spec-proposals/pull/4186/changes#r2860107511 # - # We need to fetch the `heroes` if the room name is not set. But we only need to - # get them on initial syncs (or the first time we send down the room) or if the + # We need to fetch the `heroes` if the room name is not set (taking + # care to treat an empty string as unset). But we only need to get them + # on initial syncs (or the first time we send down the room) or if the # membership has changed which may change the heroes. - if name_event_id is None and (initial or (not initial and membership_changed)): + if not room_name and (initial or membership_changed): # We need the room summary to extract the heroes from if room_membership_for_user_at_to_token.membership != Membership.JOIN: # TODO: Figure out how to get the membership summary for left/banned rooms @@ -1156,8 +1185,6 @@ async def get_room_sync_data( (EventTypes.Member, hero_user_id) for hero_user_id in hero_user_ids ] meta_room_state = list(hero_room_state) - if initial or name_changed: - meta_room_state.append((EventTypes.Name, "")) if initial or avatar_changed: meta_room_state.append((EventTypes.RoomAvatar, "")) @@ -1315,15 +1342,6 @@ async def get_room_sync_data( if required_state_filter != StateFilter.none(): required_room_state = required_state_filter.filter_state(room_state) - # Find the room name and avatar from the state - room_name: str | None = None - # TODO: Should we also check for `EventTypes.CanonicalAlias` - # (`m.room.canonical_alias`) as a fallback for the room name? see - # https://github.com/matrix-org/matrix-spec-proposals/pull/3575#discussion_r1671260153 - name_event = room_state.get((EventTypes.Name, "")) - if name_event is not None: - room_name = name_event.content.get("name") - room_avatar: str | None = None avatar_event = room_state.get((EventTypes.RoomAvatar, "")) if avatar_event is not None: diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index 94f4da0430..3cb6f8d464 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -36,7 +36,7 @@ locally_joined_rooms_gauge, ) from synapse.storage.databases.main.state_deltas import StateDelta -from synapse.types import JsonDict +from synapse.types import JsonMapping from synapse.util.duration import Duration from synapse.util.events import get_plain_text_topic_from_event_content @@ -213,7 +213,7 @@ async def _handle_deltas( ) continue - event_content: JsonDict = {} + event_content: JsonMapping = {} if delta.event_id is not None: event = await self.store.get_event(delta.event_id, allow_none=True) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index c88f703ae9..9ecfe0da0f 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -414,6 +414,15 @@ async def _wait_for_sync_for_user( context.tag = sync_label if since_token is not None: + # Work around a bug where older Synapse versions gave out tokens "from the + # future", i.e. that are ahead of the tokens persisted in the DB. This could + # also happen if a user is intentionally messing with the token so this also + # acts as sanitization/validation. + # + # If the token has positions ahead of our persisted positions in the + # database (invalid), then we simply use our max persisted position (recover + # gracefully); instead of waiting for a position that may never come around. + since_token = await self.event_sources.bound_future_token(since_token) # We need to make sure this worker has caught up with the token. If # this returns false it means we timed out waiting, and we should # just return an empty response. @@ -2227,19 +2236,23 @@ async def _generate_sync_entry_for_rooms( if block_all_room_ephemeral: ephemeral_by_room: dict[str, list[JsonDict]] = {} else: - now_token, ephemeral_by_room = await self.ephemeral_by_room( + ( + sync_result_builder.now_token, + ephemeral_by_room, + ) = await self.ephemeral_by_room( sync_result_builder, now_token=sync_result_builder.now_token, since_token=sync_result_builder.since_token, ) - sync_result_builder.now_token = now_token sticky_by_room: dict[str, list[str]] = {} if self.hs_config.experimental.msc4354_enabled: - now_token, sticky_by_room = await self.sticky_events_by_room( - sync_result_builder, now_token, since_token + ( + sync_result_builder.now_token, + sticky_by_room, + ) = await self.sticky_events_by_room( + sync_result_builder, sync_result_builder.now_token, since_token ) - sync_result_builder.now_token = now_token # 2. We check up front if anything has changed, if it hasn't then there is # no point in going further. diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 57792ea53c..a37b04494b 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -61,10 +61,10 @@ Better to retry more quickly than have workers wait around. 5 seconds is still a reasonable gap in time to not overwhelm the CPU/Database. -This matters most in cross-worker scenarios. When locks are on the same worker, when the -lock holder releases, we signal to other locks (with the same name/key) that they -should try reacquiring the lock immediately. But locks on other workers only re-check -based on their retry `_timeout_interval`. +This matters most when locks go stale as normally, when the lock holder releases, we +signal to other locks (with the same name/key) that they should try reacquiring the lock +immediately. But stale locks are never released and instead forcefully reaped behind the +scenes. """ WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION = Duration(minutes=10) diff --git a/synapse/notifier.py b/synapse/notifier.py index f1cec74462..6a057ac09f 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -832,15 +832,49 @@ async def check_for_updates( return result async def wait_for_stream_token(self, stream_token: StreamToken) -> bool: - """Wait for this worker to catch up with the given stream token.""" + """ + Wait for this worker to catch up with the given stream token. + + This is important to ensure that the worker has a proper view of the world + before trying to serve a request. For example, one worker can return a response + with some `next_batch` token, but then the next request goes to another worker + which is behind; if the worker assembles a response up to the token, it could be + missing data in the gap between where it's behind and the requested token. + + ### Inavlid future tokens + + We assume the token has already been validated/sanitized before being passed to + this function to ensure it's not some invalid future token. We consider a token + invalid, if the token has positions ahead of our persisted positions in the + database. This is important as we we don't want to wait for the stream to + advance in those cases (as it may never do so) (it's a waste of time for the + user and server). + + Previously, we would sanitize and `bound_future_token(...)` within this function + but that leads to bad patterns upstream where people can continue to use the + unbounded token. + + While it was possible for older Synapse versions to erroneously give out invalid + future tokens, this is no longer the case and its considered a Synapse + programming error if this ever happens. Validation/sanitization is still + necessary as a user can intentionally mess with numbers in the tokens being + provided. + + Args: + stream_token: The token to wait for. We assume the token has already been + validated/sanitized to ensure it's not some invalid future token (has a + stream position ahead of what is in the DB). (see details above) + + Returns: + True when this worker has caught up + False when we timed out waiting + """ current_token = self.event_sources.get_current_token() + # Return early if we are already caught up if stream_token.is_before_or_eq(current_token): return True - # Work around a bug where older Synapse versions gave out tokens "from - # the future", i.e. that are ahead of the tokens persisted in the DB. - stream_token = await self.event_sources.bound_future_token(stream_token) - + # Start waiting until we've caught up to the `stream_token` start = self.clock.time_msec() logged = False while True: @@ -850,6 +884,7 @@ async def wait_for_stream_token(self, stream_token: StreamToken) -> bool: now = self.clock.time_msec() + # Timed out if now - start > 10_000: return False diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 7cf89200a8..03dd341744 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -51,7 +51,7 @@ from synapse.storage.invite_rule import InviteRule from synapse.storage.roommember import ProfileInfo from synapse.synapse_rust.push import FilteredPushRules, PushRuleEvaluator -from synapse.types import JsonValue +from synapse.types import JsonMapping, JsonValue from synapse.types.state import StateFilter from synapse.util import unwrapFirstError from synapse.util.async_helpers import gather_results @@ -231,7 +231,7 @@ async def _get_power_levels_and_sender_level( event: EventBase, context: EventContext, event_id_to_event: Mapping[str, EventBase], - ) -> tuple[dict, int | None]: + ) -> tuple[JsonMapping, int | None]: """ Given an event and an event context, get the power level event relevant to the event and the power level of the sender of the event. diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index 1633cca884..35454c1522 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -43,7 +43,13 @@ from synapse.storage.databases.main.media_repository import ( MediaSortOrder, ) -from synapse.types import JsonDict, UserID +from synapse.types import ( + JsonDict, + MultiWriterStreamToken, + StreamKeyType, + StreamToken, + UserID, +) if TYPE_CHECKING: from synapse.server import HomeServer @@ -243,6 +249,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.server_name = hs.hostname self.replication = hs.get_replication_data_handler() + self.notifier = hs.get_notifier() async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) @@ -256,8 +263,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: # The caller is trying to get future data, which we don't allow because # we know it's an invalid state that should never happen. We could # wait until we reach the token but we might as well not waste our - # resources on that which is why `wait_for_quarantined_media_stream_id(...)` - # has assertions around this. + # resources on that. raise SynapseError( HTTPStatus.BAD_REQUEST, "The `from` token is considered invalid because it includes stream positions " @@ -268,9 +274,16 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: errcode=Codes.INVALID_PARAM, ) + # Create a `StreamToken` that's compatible with `wait_for_stream_token`. + # + # FIXME: Ideally, this endpoint would use a `StreamToken` to begin with + from_token = StreamToken.START.copy_and_replace( + StreamKeyType.QUARANTINED_MEDIA, MultiWriterStreamToken(stream=from_id) + ) + # We need to wait to ensure that our current worker is actually caught up with # the stream position, otherwise we might not return what we think we're returning. - if not await self.store.wait_for_quarantined_media_stream_id(from_id): + if not await self.notifier.wait_for_stream_token(from_token): raise SynapseError( HTTPStatus.INTERNAL_SERVER_ERROR, "Timed out while waiting for the worker serving this request to catch up to the given " @@ -280,7 +293,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: errcode=Codes.UNKNOWN, ) - to_id = await self.store.get_current_quarantined_media_stream_id() + to_id = self.store.get_current_quarantined_media_stream_id() changes = await self.store.get_quarantined_media_changes( from_id=from_id, to_id=to_id, diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py index 705d74dee1..2be5f5849d 100644 --- a/synapse/rest/client/capabilities.py +++ b/synapse/rest/client/capabilities.py @@ -77,6 +77,11 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: } } + if self.config.experimental.msc4452_enabled: + response["capabilities"]["io.element.msc4452.preview_url"] = { + "enabled": self.config.media.url_preview_enabled, + } + if self.config.experimental.msc3720_enabled: response["capabilities"]["org.matrix.msc3720.account_status"] = { "enabled": True, diff --git a/synapse/rest/client/media.py b/synapse/rest/client/media.py index 15f58acb95..c740659cdd 100644 --- a/synapse/rest/client/media.py +++ b/synapse/rest/client/media.py @@ -23,7 +23,12 @@ import logging import re -from synapse.api.errors import Codes, cs_error +from synapse.api.errors import ( + Codes, + SynapseError, + UnrecognizedRequestError, + cs_error, +) from synapse.http.server import ( HttpServer, respond_with_json, @@ -79,11 +84,17 @@ def __init__( self.clock = hs.get_clock() self.media_repo = media_repo self.media_storage = media_storage - assert self.media_repo.url_previewer is not None self.url_previewer = self.media_repo.url_previewer + self.can_respond_403 = hs.config.experimental.msc4452_enabled async def on_GET(self, request: SynapseRequest) -> None: requester = await self.auth.get_user_by_req(request) + if self.url_previewer is None: + # If we have no url_previewer then it has been disabled by the server. + if self.can_respond_403: + raise SynapseError(403, "URL Previews are disabled", Codes.FORBIDDEN) + else: + raise UnrecognizedRequestError(code=404) url = parse_string(request, "url", required=True) ts = parse_integer(request, "ts") if ts is None: @@ -299,10 +310,7 @@ async def on_GET( def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: media_repo = hs.get_media_repository() - if hs.config.media.url_preview_enabled: - PreviewURLServlet(hs, media_repo, media_repo.media_storage).register( - http_server - ) + PreviewURLServlet(hs, media_repo, media_repo.media_storage).register(http_server) MediaConfigResource(hs).register(http_server) ThumbnailResource(hs, media_repo, media_repo.media_storage).register(http_server) DownloadResource(hs, media_repo).register(http_server) diff --git a/synapse/rest/media/media_repository_resource.py b/synapse/rest/media/media_repository_resource.py index 963b9de252..354144af35 100644 --- a/synapse/rest/media/media_repository_resource.py +++ b/synapse/rest/media/media_repository_resource.py @@ -106,8 +106,7 @@ def register_servlets(http_server: HttpServer, hs: "HomeServer") -> None: ThumbnailResource(hs, media_repo, media_repo.media_storage).register( http_server ) - if hs.config.media.url_preview_enabled: - PreviewUrlResource(hs, media_repo, media_repo.media_storage).register( - http_server - ) + PreviewUrlResource(hs, media_repo, media_repo.media_storage).register( + http_server + ) MediaConfigResource(hs).register(http_server) diff --git a/synapse/rest/media/preview_url_resource.py b/synapse/rest/media/preview_url_resource.py index bfeff2179b..5a7bdf38f0 100644 --- a/synapse/rest/media/preview_url_resource.py +++ b/synapse/rest/media/preview_url_resource.py @@ -23,6 +23,7 @@ import re from typing import TYPE_CHECKING +from synapse.api.errors import Codes, SynapseError, UnrecognizedRequestError from synapse.http.server import respond_with_json_bytes from synapse.http.servlet import RestServlet, parse_integer, parse_string from synapse.http.site import SynapseRequest @@ -65,12 +66,17 @@ def __init__( self.clock = hs.get_clock() self.media_repo = media_repo self.media_storage = media_storage - assert self.media_repo.url_previewer is not None self.url_previewer = self.media_repo.url_previewer + self.can_respond_403 = hs.config.experimental.msc4452_enabled async def on_GET(self, request: SynapseRequest) -> None: - # XXX: if get_user_by_req fails, what should we do in an async render? requester = await self.auth.get_user_by_req(request) + if self.url_previewer is None: + # If we have no url_previewer then it has been disabled by the server. + if self.can_respond_403: + raise SynapseError(403, "URL Previews are disabled", Codes.FORBIDDEN) + else: + raise UnrecognizedRequestError(code=404) url = parse_string(request, "url", required=True) ts = parse_integer(request, "ts", default=self.clock.time_msec()) og = await self.url_previewer.preview(url, requester.user, ts) diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 2f0e3f2c3e..b22f91b37c 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -37,7 +37,8 @@ from synapse.api.constants import EventTypes from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, StateResolutionVersions -from synapse.events import EventBase, FrozenEventVMSC4242 +from synapse.events import EventBase +from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.events.snapshot import ( EventContext, UnpersistedEventContext, @@ -315,7 +316,7 @@ async def calculate_context_info( # might redundantly recalculate the state for this event later.) prev_event_ids = frozenset( event.prev_state_events - if isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event) else event.prev_event_ids() ) incomplete_prev_events = await self.store.get_partial_state_events( diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index 7cc6a39639..62e84f5ac5 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -34,6 +34,7 @@ Generator, Generic, Iterable, + Sequence, TypeVar, cast, ) @@ -46,7 +47,14 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.errors import SynapseError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS -from synapse.events import EventBase, FrozenEventVMSC4242, event_exists_in_state_dag +from synapse.events import ( + EventBase, + event_exists_in_state_dag, +) +from synapse.events.py_protocol import ( + MSC4242Event, + all_supports_msc4242_state_dag, +) from synapse.events.snapshot import EventContext, EventPersistencePair from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable @@ -637,7 +645,6 @@ async def _persist_event_batch( # Get the room version for the first event. This room version is the same for all events # as events_and_contexts is all for one room. assert len(events_and_contexts) > 0 - room_version = events_and_contexts[0][0].room_version for chunk in chunks: # We can't easily parallelize these since different chunks @@ -648,22 +655,18 @@ async def _persist_event_batch( new_state_dag_extrems = None if not backfilled: - if room_version.msc4242_state_dags: + if all_supports_msc4242_state_dag(chunk): with Measure( self._clock, name="_process_state_dag_forward_extremities_and_state_delta", server_name=self.server_name, ): - assert all( - isinstance(ev, FrozenEventVMSC4242) for ev, _ in chunk - ) ( new_forward_extremities, # for prev_events state_delta_for_room, # for state groups new_state_dag_extrems, # for prev_state_events ) = await self._process_state_dag_forward_extremities_and_state_delta( - room_id, - cast(list[tuple[FrozenEventVMSC4242, EventContext]], chunk), + room_id, chunk ) else: with Measure( @@ -840,7 +843,7 @@ async def _calculate_new_forward_extremities_and_state_delta( async def _process_state_dag_forward_extremities_and_state_delta( self, room_id: str, - event_contexts: list[tuple[FrozenEventVMSC4242, EventContext]], + event_contexts: Sequence[tuple[MSC4242Event, EventContext]], ) -> tuple[set[str] | None, DeltaState | None, set[str] | None]: """Process the forwards extremities for state DAG rooms. Returns: @@ -933,7 +936,7 @@ async def _calculate_new_state_dag_extremities( self, room_id: str, existing_fwd_extrems: frozenset[str], - event_contexts: list[tuple[FrozenEventVMSC4242, EventContext]], + event_contexts: Sequence[tuple[MSC4242Event, EventContext]], ) -> set[str]: """Calculate the new state dag forward extremities. Modifies existing_fwd_extrems. diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 5f6b03a988..5aab0067fc 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -48,12 +48,12 @@ from synapse.api.room_versions import RoomVersions from synapse.events import ( EventBase, - FrozenEventVMSC4242, StrippedStateEvent, event_exists_in_state_dag, is_creator, relation_from_event, ) +from synapse.events.py_protocol import MSC4242Event, supports_msc4242_state_dag from synapse.events.snapshot import EventPersistencePair from synapse.events.utils import parse_stripped_state_event from synapse.logging.opentracing import trace @@ -2897,10 +2897,7 @@ def _update_metadata_tables_txn( self._handle_event_relations(txn, event) - if event.room_version.msc4242_state_dags and event_exists_in_state_dag( - event - ): - assert isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event) and event_exists_in_state_dag(event): self._store_state_dag_edges(txn, event) # Store the labels for this event. @@ -2980,7 +2977,7 @@ def local_prefill() -> None: txn.call_after(local_prefill) def _store_state_dag_edges( - self, txn: LoggingTransaction, event: FrozenEventVMSC4242 + self, txn: LoggingTransaction, event: MSC4242Event ) -> None: # the create event has no edge but we still need to persist it as get_state_dag just # yanks all rows in this table. It's a bit gross to store NULL as the prev_state_event_id diff --git a/synapse/storage/databases/main/lock.py b/synapse/storage/databases/main/lock.py index dd49f98366..decb74e994 100644 --- a/synapse/storage/databases/main/lock.py +++ b/synapse/storage/databases/main/lock.py @@ -53,9 +53,9 @@ _RENEWAL_INTERVAL = Duration(seconds=30) # How long before an acquired lock times out. -_LOCK_TIMEOUT_MS = 2 * 60 * 1000 +_LOCK_TIMEOUT = Duration(minutes=2) -_LOCK_REAP_INTERVAL = Duration(milliseconds=_LOCK_TIMEOUT_MS / 10.0) +_LOCK_REAP_INTERVAL = Duration(milliseconds=_LOCK_TIMEOUT.as_millis() / 10.0) class LockStore(SQLBaseStore): @@ -63,7 +63,7 @@ class LockStore(SQLBaseStore): Locks are identified by a name and key. A lock is acquired by inserting into the `worker_locks` table if a) there is no existing row for the name/key or - b) the existing row has a `last_renewed_ts` older than `_LOCK_TIMEOUT_MS`. + b) the existing row has a `last_renewed_ts` older than `_LOCK_TIMEOUT`. When a lock is taken out the instance inserts a random `token`, the instance that holds that token holds the lock until it drops (or times out). @@ -182,7 +182,7 @@ def _try_acquire_lock_txn(txn: LoggingTransaction) -> bool: self._instance_name, token, now, - now - _LOCK_TIMEOUT_MS, + now - _LOCK_TIMEOUT.as_millis(), ), ) @@ -340,7 +340,9 @@ async def _reap_stale_read_write_locks(self) -> None: """ def reap_stale_read_write_locks_txn(txn: LoggingTransaction) -> None: - txn.execute(delete_sql, (self.clock.time_msec() - _LOCK_TIMEOUT_MS,)) + txn.execute( + delete_sql, (self.clock.time_msec() - _LOCK_TIMEOUT.as_millis(),) + ) if txn.rowcount: logger.info("Reaped %d stale locks", txn.rowcount) @@ -489,7 +491,7 @@ async def is_still_valid(self) -> bool: ) return ( last_renewed_ts is not None - and self._clock.time_msec() - _LOCK_TIMEOUT_MS < last_renewed_ts + and self._clock.time_msec() - _LOCK_TIMEOUT.as_millis() < last_renewed_ts ) async def __aenter__(self) -> None: diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 23d5b457eb..8fd41ccb7e 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -2568,9 +2568,6 @@ async def user_delete_access_tokens_for_devices( def user_delete_access_tokens_for_devices_txn( txn: LoggingTransaction, batch_device_ids: StrCollection ) -> list[tuple[str, int, str | None]]: - # Delete access tokens first, before refresh tokens. - # This ensures we can capture the deleted access tokens for cache invalidation - # before any CASCADE deletes occur. clause, args = make_in_list_sql_clause( txn.database_engine, "device_id", batch_device_ids ) @@ -2589,6 +2586,9 @@ def user_delete_access_tokens_for_devices_txn( self.get_user_by_access_token, [(t[0],) for t in tokens_and_devices], ) + # Delete access tokens first, before refresh tokens. + # This ensures we can capture the deleted access tokens for cache invalidation + # before any CASCADE deletes occur. self.db_pool.simple_delete_many_txn( txn, table="refresh_tokens", diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index a0c42082f0..95aa2cb7dc 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -62,12 +62,12 @@ from synapse.storage.util.id_generators import IdGenerator, MultiWriterIdGenerator from synapse.types import ( JsonDict, + MultiWriterStreamToken, RetentionPolicy, StrCollection, ThirdPartyInstanceID, ) from synapse.util.caches.descriptors import cached, cachedList -from synapse.util.duration import Duration from synapse.util.json import json_encoder from synapse.util.stringutils import MXC_REGEX @@ -1302,7 +1302,15 @@ def _get_media_ids_by_user_txn( return local_media_ids - async def get_current_quarantined_media_stream_id(self) -> int: + def get_quarantined_media_stream_token(self) -> MultiWriterStreamToken: + return MultiWriterStreamToken.from_generator( + self._quarantined_media_changes_id_gen + ) + + def get_quarantined_media_stream_id_generator(self) -> MultiWriterIdGenerator: + return self._quarantined_media_changes_id_gen + + def get_current_quarantined_media_stream_id(self) -> int: """Gets the position of the quarantined media changes stream. Returns: @@ -1318,74 +1326,6 @@ async def get_max_allocated_quarantined_media_stream_id(self) -> int: """ return await self._quarantined_media_changes_id_gen.get_max_allocated_token() - async def wait_for_quarantined_media_stream_id(self, target_id: int) -> bool: - """Waits until the quarantined media changes stream reaches the given stream ID. - - See https://github.com/element-hq/synapse/pull/19644 for more details. - - TODO: Replace function and call sites with https://github.com/element-hq/synapse/pull/19644 - - Args: - target_id: The stream ID to wait for. - - Returns: - True when caught up to the target stream ID. - False when timing out while waiting. - """ - # We ideally would use something like `wait_for_stream_position` in the meantime, - # but that short circuits if the instance name matches the current instance name. - # Doing so means that if *another* writer is actually leading the to_id, then we'll - # assume that we're caught up when we aren't. - # - # NOTE: Because this is implemented to wait for stream positions by integer ID, - # we're technically waiting for *all* workers to catch up rather than just waiting - # for *our* worker to catch up. This is okay for now because the quarantined media - # stream should be pretty fast to update, and if it's not then the only thing we're - # affecting is an admin API that probably has a tool automatically retrying requests - # anyway. https://github.com/element-hq/synapse/pull/19644 does the waiting properly - # so this should be replaced by that (or similar). - - # Get the minimum shared position/ID across all workers - current_id = self._quarantined_media_changes_id_gen.get_current_token() - if current_id >= target_id: - return True # nothing to wait for: we're already caught up. - - # "This should never happen". Tokens we hand out via the API should exist. If they - # don't, then we're in a bad state and need to explode. - max_persisted_position = ( - await self._quarantined_media_changes_id_gen.get_max_allocated_token() - ) - assert max_persisted_position >= target_id, ( - f"Unable to wait for invalid future token (token={target_id} has positions " - f"ahead of our max persisted position={max_persisted_position})" - ) - - # Start waiting until we've caught up to the `stream_token` - start = self.clock.time_msec() - logged = False - while True: - # Like above, get the minimum shared ID across all workers - current_id = self._quarantined_media_changes_id_gen.get_current_token() - if current_id >= target_id: - return True - - now = self.clock.time_msec() - - # Timed out - if now - start > 10_000: - return False - - if not logged: - logger.info( - "Waiting for current token to reach %s; currently at %s", - target_id, - current_id, - ) - logged = True - - # TODO: be better - await self.clock.sleep(Duration(milliseconds=500)) - async def get_quarantined_media_changes( self, *, from_id: int, to_id: int, limit: int ) -> list[QuarantinedMediaUpdate]: diff --git a/synapse/streams/events.py b/synapse/streams/events.py index d2720fb959..36490fcb35 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -85,6 +85,7 @@ def get_current_token(self) -> StreamToken: ) thread_subscriptions_key = self.store.get_max_thread_subscriptions_stream_id() sticky_events_key = self.store.get_max_sticky_events_stream_id() + quarantined_media_key = self.store.get_quarantined_media_stream_token() token = StreamToken( room_key=self.sources.room.get_current_key(), @@ -100,6 +101,7 @@ def get_current_token(self) -> StreamToken: un_partial_stated_rooms_key=un_partial_stated_rooms_key, thread_subscriptions_key=thread_subscriptions_key, sticky_events_key=sticky_events_key, + quarantined_media_key=quarantined_media_key, ) return token @@ -128,6 +130,7 @@ async def bound_future_token(self, token: StreamToken) -> StreamToken: StreamKeyType.UN_PARTIAL_STATED_ROOMS: self.store.get_un_partial_stated_rooms_id_generator(), StreamKeyType.THREAD_SUBSCRIPTIONS: self.store.get_thread_subscriptions_stream_id_generator(), StreamKeyType.STICKY_EVENTS: self.store.get_sticky_events_stream_id_generator(), + StreamKeyType.QUARANTINED_MEDIA: self.store.get_quarantined_media_stream_id_generator(), } for _, key in StreamKeyType.__members__.items(): @@ -149,6 +152,16 @@ async def bound_future_token(self, token: StreamToken) -> StreamToken: ].get_max_allocated_token() if max_token < token_value.get_max_stream_pos(): + # Log *something* as we consider this as a Synapse programming error + # (assuming no malicious user manipulation of the token) (we shouldn't be + # handing out future tokens). + # + # We don't assert as the whole point of bounding is so that we can recover + # gracefully. + # + # Old versions of Synapse could advance streams without persisting anything in + # the DB (fixed in https://github.com/element-hq/synapse/pull/17229) and on + # restart, those updates would be lost. logger.error( "Bounding token from the future '%s': token: %s, bound: %s", key, diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index 138cdde0be..a6489ffc2c 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -10,7 +10,7 @@ # See the GNU Affero General Public License for more details: # . -from typing import Any, Mapping +from typing import Any, Iterator, Mapping from synapse.types import JsonDict, JsonMapping @@ -213,3 +213,12 @@ class Unsigned: def for_event(self) -> JsonDict: ... """Return a dict of all unsigned fields, including those only kept in memory, suitable for inclusion in an event.""" + +class JsonObject(Mapping[str, Any]): + """Immutable JSON object mapping.""" + + def __init__(self, content_dict: JsonMapping | None = None): ... + def __len__(self) -> int: ... + def __getitem__(self, key: str) -> Any: ... + def __iter__(self) -> Iterator[str]: ... + def __eq__(self, other: object) -> bool: ... diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 02889795bb..8537a63bde 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -685,6 +685,9 @@ def is_before_or_eq(self, other_token: Self) -> bool: def bound_stream_token(self, max_stream: int) -> "Self": """Bound the stream positions to a maximum value""" + # Shortcut if we're already under the bound + if self.get_max_stream_pos() <= max_stream: + return self min_pos = min(self.stream, max_stream) return type(self)( @@ -1057,6 +1060,7 @@ class StreamKeyType(Enum): UN_PARTIAL_STATED_ROOMS = "un_partial_stated_rooms_key" THREAD_SUBSCRIPTIONS = "thread_subscriptions_key" STICKY_EVENTS = "sticky_events_key" + QUARANTINED_MEDIA = "quarantined_media_key" @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -1064,7 +1068,7 @@ class StreamToken: """A collection of keys joined together by underscores in the following order and which represent the position in their respective streams. - ex. `s2633508_17_338_6732159_1082514_541479_274711_265584_1_379_4242` + ex. `s2633508_17_338_6732159_1082514_541479_274711_265584_1_379_4242_4141_4343` 1. `room_key`: `s2633508` which is a `RoomStreamToken` - `RoomStreamToken`'s can also look like `t426-2633508` or `m56~2.58~3.59` - See the docstring for `RoomStreamToken` for more details. @@ -1079,12 +1083,13 @@ class StreamToken: 10. `un_partial_stated_rooms_key`: `379` 11. `thread_subscriptions_key`: 4242 12. `sticky_events_key`: 4141 + 13. `quarantined_media_key`: 4343 You can see how many of these keys correspond to the various fields in a "/sync" response: ```json { - "next_batch": "s12_4_0_1_1_1_1_4_1_1", + "next_batch": "s12_4_0_1_1_1_1_4_1_1_1_1_1", "presence": { "events": [] }, @@ -1096,7 +1101,7 @@ class StreamToken: "!QrZlfIDQLNLdZHqTnt:hs1": { "timeline": { "events": [], - "prev_batch": "s10_4_0_1_1_1_1_4_1_1", + "prev_batch": "s10_4_0_1_1_1_1_4_1_1_1_1_1", "limited": false }, "state": { @@ -1139,6 +1144,9 @@ class StreamToken: un_partial_stated_rooms_key: int thread_subscriptions_key: int sticky_events_key: int + quarantined_media_key: MultiWriterStreamToken = attr.ib( + validator=attr.validators.instance_of(MultiWriterStreamToken) + ) _SEPARATOR = "_" START: ClassVar["StreamToken"] @@ -1168,6 +1176,7 @@ async def from_string(cls, store: "DataStore", string: str) -> "StreamToken": un_partial_stated_rooms_key, thread_subscriptions_key, sticky_events_key, + quarantined_media_key, ) = keys return cls( @@ -1185,6 +1194,9 @@ async def from_string(cls, store: "DataStore", string: str) -> "StreamToken": un_partial_stated_rooms_key=int(un_partial_stated_rooms_key), thread_subscriptions_key=int(thread_subscriptions_key), sticky_events_key=int(sticky_events_key), + quarantined_media_key=await MultiWriterStreamToken.parse( + store, quarantined_media_key + ), ) except CancelledError: raise @@ -1209,6 +1221,7 @@ async def to_string(self, store: "DataStore") -> str: str(self.un_partial_stated_rooms_key), str(self.thread_subscriptions_key), str(self.sticky_events_key), + await self.quarantined_media_key.to_string(store), ] ) @@ -1238,6 +1251,12 @@ def copy_and_advance(self, key: StreamKeyType, new_value: Any) -> "StreamToken": self.device_list_key.copy_and_advance(new_value), ) return new_token + elif key == StreamKeyType.QUARANTINED_MEDIA: + new_token = self.copy_and_replace( + StreamKeyType.QUARANTINED_MEDIA, + self.quarantined_media_key.copy_and_advance(new_value), + ) + return new_token new_token = self.copy_and_replace(key, new_value) new_id = new_token.get_field(key) @@ -1260,6 +1279,7 @@ def get_field( key: Literal[ StreamKeyType.RECEIPT, StreamKeyType.DEVICE_LIST, + StreamKeyType.QUARANTINED_MEDIA, ], ) -> MultiWriterStreamToken: ... @@ -1331,7 +1351,8 @@ def __str__(self) -> str: f"account_data: {self.account_data_key}, push_rules: {self.push_rules_key}, " f"to_device: {self.to_device_key}, device_list: {self.device_list_key}, " f"groups: {self.groups_key}, un_partial_stated_rooms: {self.un_partial_stated_rooms_key}," - f"thread_subscriptions: {self.thread_subscriptions_key}, sticky_events: {self.sticky_events_key})" + f"thread_subscriptions: {self.thread_subscriptions_key}, sticky_events: {self.sticky_events_key}" + f"quarantined_media: {self.quarantined_media_key})" ) @@ -1348,6 +1369,7 @@ def __str__(self) -> str: un_partial_stated_rooms_key=0, thread_subscriptions_key=0, sticky_events_key=0, + quarantined_media_key=MultiWriterStreamToken(stream=0), ) diff --git a/synapse/util/events.py b/synapse/util/events.py index 19eca1c1ae..e7c1c83a37 100644 --- a/synapse/util/events.py +++ b/synapse/util/events.py @@ -17,7 +17,7 @@ from pydantic import Field, StrictStr, ValidationError, field_validator -from synapse.types import JsonDict +from synapse.types import JsonMapping from synapse.util.pydantic_models import ParseModel from synapse.util.stringutils import random_string @@ -103,7 +103,7 @@ def ignore_invalid_m_topic(cls, m_topic: Any) -> MTopic | None: return None -def get_plain_text_topic_from_event_content(content: JsonDict) -> str | None: +def get_plain_text_topic_from_event_content(content: JsonMapping) -> str | None: """ Given the `content` of an `m.room.topic` event, returns the plain-text topic representation. Prefers pulling plain-text from the newer `m.topic` field if diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index 0bc27410c6..92c03690f2 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -23,6 +23,8 @@ from immutabledict import immutabledict +from synapse.synapse_rust.events import JsonObject + def freeze(o: Any) -> Any: if isinstance(o, dict): @@ -31,6 +33,9 @@ def freeze(o: Any) -> Any: if isinstance(o, immutabledict): return o + if isinstance(o, JsonObject): + return o + if isinstance(o, (bytes, str)): return o diff --git a/synapse/util/json.py b/synapse/util/json.py index b1091704a8..8f8d731c6d 100644 --- a/synapse/util/json.py +++ b/synapse/util/json.py @@ -20,24 +20,30 @@ from immutabledict import immutabledict +from synapse.synapse_rust.events import JsonObject + def _reject_invalid_json(val: Any) -> None: """Do not allow Infinity, -Infinity, or NaN values in JSON.""" raise ValueError("Invalid JSON value: '%s'" % val) -def _handle_immutabledict(obj: Any) -> dict[Any, Any]: - """Helper for json_encoder. Makes immutabledicts serializable by returning - the underlying dict +def _handle_extra_mappings(obj: Any) -> dict[Any, Any]: + """Helper for json_encoder. Makes immutabledicts and JsonObjects + serializable """ - if type(obj) is immutabledict: - # fishing the protected dict out of the object is a bit nasty, - # but we don't really want the overhead of copying the dict. - try: - # Safety: we catch the AttributeError immediately below. - return obj._dict - except AttributeError: - # If all else fails, resort to making a copy of the immutabledict + match obj: + case immutabledict(): + # fishing the protected dict out of the object is a bit nasty, + # but we don't really want the overhead of copying the dict. + try: + # Safety: we catch the AttributeError immediately below. + return obj._dict + except AttributeError: + # If all else fails, resort to making a copy of the immutabledict + return dict(obj) + case JsonObject(): + # Just convert to a dict. return dict(obj) raise TypeError( "Object of type %s is not JSON serializable" % obj.__class__.__name__ @@ -49,7 +55,7 @@ def _handle_immutabledict(obj: Any) -> dict[Any, Any]: # * produces valid JSON (no NaNs etc) # * reduces redundant whitespace json_encoder = json.JSONEncoder( - allow_nan=False, separators=(",", ":"), default=_handle_immutabledict + allow_nan=False, separators=(",", ":"), default=_handle_extra_mappings ) # Create a custom decoder to reject Python extensions to JSON. diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 7742a06b4c..8c17e98c0d 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -36,7 +36,7 @@ from synapse.util.frozenutils import freeze from tests import unittest -from tests.events.test_utils import MockEvent +from tests.test_utils.event_builders import make_test_event user_id = UserID.from_string("@test_user:test") user2_id = UserID.from_string("@test_user2:test") @@ -134,18 +134,22 @@ def test_limits_are_applied(self) -> None: def test_definition_types_works_with_literals(self) -> None: definition = {"types": ["m.room.message", "org.matrix.foo.bar"]} - event = MockEvent(sender="@foo:bar", type="m.room.message", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="m.room.message", room_id="!foo:bar" + ) self.assertTrue(Filter(self.hs, definition)._check(event)) def test_definition_types_works_with_wildcards(self) -> None: definition = {"types": ["m.*", "org.matrix.foo.bar"]} - event = MockEvent(sender="@foo:bar", type="m.room.message", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="m.room.message", room_id="!foo:bar" + ) self.assertTrue(Filter(self.hs, definition)._check(event)) def test_definition_types_works_with_unknowns(self) -> None: definition = {"types": ["m.room.message", "org.matrix.foo.bar"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="now.for.something.completely.different", room_id="!foo:bar", @@ -154,19 +158,23 @@ def test_definition_types_works_with_unknowns(self) -> None: def test_definition_not_types_works_with_literals(self) -> None: definition = {"not_types": ["m.room.message", "org.matrix.foo.bar"]} - event = MockEvent(sender="@foo:bar", type="m.room.message", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="m.room.message", room_id="!foo:bar" + ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_not_types_works_with_wildcards(self) -> None: definition = {"not_types": ["m.room.message", "org.matrix.*"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="org.matrix.custom.event", room_id="!foo:bar" ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_not_types_works_with_unknowns(self) -> None: definition = {"not_types": ["m.*", "org.*"]} - event = MockEvent(sender="@foo:bar", type="com.nom.nom.nom", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="com.nom.nom.nom", room_id="!foo:bar" + ) self.assertTrue(Filter(self.hs, definition)._check(event)) def test_definition_not_types_takes_priority_over_types(self) -> None: @@ -174,33 +182,35 @@ def test_definition_not_types_takes_priority_over_types(self) -> None: "not_types": ["m.*", "org.*"], "types": ["m.room.message", "m.room.topic"], } - event = MockEvent(sender="@foo:bar", type="m.room.topic", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="m.room.topic", room_id="!foo:bar" + ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_senders_works_with_literals(self) -> None: definition = {"senders": ["@flibble:wibble"]} - event = MockEvent( + event = make_test_event( sender="@flibble:wibble", type="com.nom.nom.nom", room_id="!foo:bar" ) self.assertTrue(Filter(self.hs, definition)._check(event)) def test_definition_senders_works_with_unknowns(self) -> None: definition = {"senders": ["@flibble:wibble"]} - event = MockEvent( + event = make_test_event( sender="@challenger:appears", type="com.nom.nom.nom", room_id="!foo:bar" ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_not_senders_works_with_literals(self) -> None: definition = {"not_senders": ["@flibble:wibble"]} - event = MockEvent( + event = make_test_event( sender="@flibble:wibble", type="com.nom.nom.nom", room_id="!foo:bar" ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_not_senders_works_with_unknowns(self) -> None: definition = {"not_senders": ["@flibble:wibble"]} - event = MockEvent( + event = make_test_event( sender="@challenger:appears", type="com.nom.nom.nom", room_id="!foo:bar" ) self.assertTrue(Filter(self.hs, definition)._check(event)) @@ -210,21 +220,21 @@ def test_definition_not_senders_takes_priority_over_senders(self) -> None: "not_senders": ["@misspiggy:muppets"], "senders": ["@kermit:muppets", "@misspiggy:muppets"], } - event = MockEvent( + event = make_test_event( sender="@misspiggy:muppets", type="m.room.topic", room_id="!foo:bar" ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_rooms_works_with_literals(self) -> None: definition = {"rooms": ["!secretbase:unknown"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown" ) self.assertTrue(Filter(self.hs, definition)._check(event)) def test_definition_rooms_works_with_unknowns(self) -> None: definition = {"rooms": ["!secretbase:unknown"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!anothersecretbase:unknown", @@ -233,7 +243,7 @@ def test_definition_rooms_works_with_unknowns(self) -> None: def test_definition_not_rooms_works_with_literals(self) -> None: definition = {"not_rooms": ["!anothersecretbase:unknown"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!anothersecretbase:unknown", @@ -242,7 +252,7 @@ def test_definition_not_rooms_works_with_literals(self) -> None: def test_definition_not_rooms_works_with_unknowns(self) -> None: definition = {"not_rooms": ["!secretbase:unknown"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!anothersecretbase:unknown", @@ -254,7 +264,7 @@ def test_definition_not_rooms_takes_priority_over_rooms(self) -> None: "not_rooms": ["!secretbase:unknown"], "rooms": ["!secretbase:unknown"], } - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown" ) self.assertFalse(Filter(self.hs, definition)._check(event)) @@ -268,7 +278,7 @@ def test_definition_combined_event(self) -> None: "types": ["m.room.message", "muppets.kermit.*"], "not_types": ["muppets.misspiggy.*"], } - event = MockEvent( + event = make_test_event( sender="@kermit:muppets", # yup type="m.room.message", # yup room_id="!stage:unknown", # yup @@ -284,7 +294,7 @@ def test_definition_combined_event_bad_sender(self) -> None: "types": ["m.room.message", "muppets.kermit.*"], "not_types": ["muppets.misspiggy.*"], } - event = MockEvent( + event = make_test_event( sender="@misspiggy:muppets", # nope type="m.room.message", # yup room_id="!stage:unknown", # yup @@ -300,7 +310,7 @@ def test_definition_combined_event_bad_room(self) -> None: "types": ["m.room.message", "muppets.kermit.*"], "not_types": ["muppets.misspiggy.*"], } - event = MockEvent( + event = make_test_event( sender="@kermit:muppets", # yup type="m.room.message", # yup room_id="!piggyshouse:muppets", # nope @@ -316,7 +326,7 @@ def test_definition_combined_event_bad_type(self) -> None: "types": ["m.room.message", "muppets.kermit.*"], "not_types": ["muppets.misspiggy.*"], } - event = MockEvent( + event = make_test_event( sender="@kermit:muppets", # yup type="muppets.misspiggy.kisses", # nope room_id="!stage:unknown", # yup @@ -325,7 +335,7 @@ def test_definition_combined_event_bad_type(self) -> None: def test_filter_labels(self) -> None: definition = {"org.matrix.labels": ["#fun"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -334,7 +344,7 @@ def test_filter_labels(self) -> None: self.assertTrue(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -344,7 +354,7 @@ def test_filter_labels(self) -> None: self.assertFalse(Filter(self.hs, definition)._check(event)) # check it works with frozen dictionaries too - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -354,7 +364,7 @@ def test_filter_labels(self) -> None: def test_filter_not_labels(self) -> None: definition = {"org.matrix.not_labels": ["#fun"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -363,7 +373,7 @@ def test_filter_not_labels(self) -> None: self.assertFalse(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -375,7 +385,7 @@ def test_filter_not_labels(self) -> None: @unittest.override_config({"experimental_features": {"msc3874_enabled": True}}) def test_filter_rel_type(self) -> None: definition = {"org.matrix.msc3874.rel_types": ["m.thread"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -384,7 +394,7 @@ def test_filter_rel_type(self) -> None: self.assertFalse(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -393,7 +403,7 @@ def test_filter_rel_type(self) -> None: self.assertFalse(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -405,7 +415,7 @@ def test_filter_rel_type(self) -> None: @unittest.override_config({"experimental_features": {"msc3874_enabled": True}}) def test_filter_not_rel_type(self) -> None: definition = {"org.matrix.msc3874.not_rel_types": ["m.thread"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -414,7 +424,7 @@ def test_filter_not_rel_type(self) -> None: self.assertFalse(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -423,7 +433,7 @@ def test_filter_not_rel_type(self) -> None: self.assertTrue(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -494,7 +504,9 @@ def test_filter_room_state_match(self) -> None: user_id=user_id, user_filter=user_filter_json ) ) - event = MockEvent(sender="@foo:bar", type="m.room.topic", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="m.room.topic", room_id="!foo:bar" + ) events = [event] user_filter = self.get_success( @@ -511,7 +523,7 @@ def test_filter_room_state_no_match(self) -> None: user_id=user_id, user_filter=user_filter_json ) ) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="org.matrix.custom.event", room_id="!foo:bar" ) events = [event] @@ -542,14 +554,14 @@ def test_filter_rooms(self) -> None: def test_filter_relations(self) -> None: events = [ # An event without a relation. - MockEvent( + make_test_event( event_id="$no_relation", sender="@foo:bar", type="org.matrix.custom.event", room_id="!foo:bar", ), # An event with a relation. - MockEvent( + make_test_event( event_id="$with_relation", sender="@foo:bar", type="org.matrix.custom.event", diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py index 334ff64bc2..e4621b1c2b 100644 --- a/tests/crypto/test_event_signing.py +++ b/tests/crypto/test_event_signing.py @@ -34,10 +34,11 @@ event_needs_resigning, resign_event, ) -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.types import JsonDict from tests import unittest +from tests.test_utils.event_builders import make_test_event # Perform these tests using given secret key so we get entirely deterministic # signatures output that we can test against. @@ -62,6 +63,7 @@ def test_sign_minimal(self) -> None: "origin_server_ts": 1000000, "signatures": {}, "type": "X", + "content": {}, "unsigned": {"age_ts": 1000000}, } @@ -69,12 +71,12 @@ def test_sign_minimal(self) -> None: RoomVersions.V1, event_dict, HOSTNAME, self.signing_key ) - event = make_event_from_dict(event_dict) + event = make_test_event(event_dict) self.assertTrue(hasattr(event, "hashes")) self.assertIn("sha256", event.hashes) self.assertEqual( - event.hashes["sha256"], "A6Nco6sqoy18PPfPDVdYvoowfc0PVBk9g9OiyT3ncRM" + event.hashes["sha256"], "mq4QfPPpC+QsBd6eqfVsmJIEz8uvMSVK0+AU67PLESk" ) self.assertTrue(hasattr(event, "signatures")) @@ -82,8 +84,8 @@ def test_sign_minimal(self) -> None: self.assertIn(KEY_NAME, event.signatures["domain"]) self.assertEqual( event.signatures[HOSTNAME][KEY_NAME], - "PBc48yDVszWB9TRaB/+CZC1B+pDAC10F8zll006j+NN" - "fe4PEMWcVuLaG63LFTK9e4rwJE8iLZMPtCKhDTXhpAQ", + "18rGIkd4JJXxw9m+1j3BtN+TmqmLip4VHvFbyXLngpB" + "LXOqbxlQViQABRzep2cODQ2aa5FnFgz+Llt2P03WiAw", ) def test_sign_message(self) -> None: @@ -102,7 +104,7 @@ def test_sign_message(self) -> None: RoomVersions.V1, event_dict, HOSTNAME, self.signing_key ) - event = make_event_from_dict(event_dict) + event = make_test_event(event_dict) self.assertTrue(hasattr(event, "hashes")) self.assertIn("sha256", event.hashes) @@ -140,7 +142,7 @@ def test_resign(self) -> None: add_hashes_and_signatures( RoomVersions.V1, event_dict, HOSTNAME, self.signing_key ) - event = make_event_from_dict(event_dict) + event = make_test_event(event_dict) self.assertIn(HOSTNAME, event.signatures) self.assertIn(KEY_NAME, event.signatures[HOSTNAME]) signature = event.signatures[HOSTNAME][KEY_NAME] @@ -170,7 +172,7 @@ def test_resign(self) -> None: "signatures": {}, "unsigned": {"age_ts": 1000000}, } - event = make_event_from_dict(event_dict) + event = make_test_event(event_dict) resigned_event = resign_event(event, HOSTNAME, signing_key_2) self.assertIn(HOSTNAME, resigned_event["signatures"]) self.assertIn(key_name_2, resigned_event["signatures"][HOSTNAME]) @@ -187,7 +189,7 @@ def test_event_needs_resigning(self) -> None: "unsigned": {"age_ts": 1000000}, } internal_metadata: JsonDict = {} - event_that_needs_resigning = make_event_from_dict( + event_that_needs_resigning = make_test_event( event_that_needs_resigning_dict, RoomVersions.V1, internal_metadata, @@ -206,7 +208,7 @@ class TestCase(TypedDict): events_that_dont_need_resigning: list[TestCase] = [ { "name": "sender domain isn't ours", - "event": make_event_from_dict( + "event": make_test_event( {**event_that_needs_resigning_dict, "sender": "@u:somewhereelse"}, RoomVersions.V1, internal_metadata, @@ -214,7 +216,7 @@ class TestCase(TypedDict): }, { "name": "already signed with this key", - "event": make_event_from_dict( + "event": make_test_event( { **event_that_needs_resigning_dict, "signatures": { diff --git a/tests/events/test_auto_accept_invites.py b/tests/events/test_auto_accept_invites.py index e0ebdf0bca..632d1dc4f6 100644 --- a/tests/events/test_auto_accept_invites.py +++ b/tests/events/test_auto_accept_invites.py @@ -33,7 +33,6 @@ from synapse.config._base import RootConfig from synapse.config.auto_accept_invites import AutoAcceptInvitesConfig from synapse.events.auto_accept_invites import InviteAutoAccepter -from synapse.federation.federation_base import event_from_pdu_json from synapse.handlers.sync import JoinedSyncResult, SyncRequestKey from synapse.module_api import ModuleApi from synapse.rest import admin @@ -43,6 +42,7 @@ from synapse.util.clock import Clock from tests.handlers.test_sync import generate_sync_config +from tests.test_utils.event_builders import make_test_pdu_event from tests.unittest import ( FederatingHomeserverTestCase, HomeserverTestCase, @@ -182,7 +182,7 @@ def test_invite_from_remote_user(self) -> None: ) room_version = self.get_success(self.store.get_room_version(room_id)) - invite_event = event_from_pdu_json( + invite_event = make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, @@ -308,7 +308,7 @@ def test_accept_invite_local_user( remote_server = "otherserver" remote_user = "@otheruser:" + remote_server - invite_event = event_from_pdu_json( + invite_event = make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, diff --git a/tests/events/test_py_protocol.py b/tests/events/test_py_protocol.py new file mode 100644 index 0000000000..306e3c1704 --- /dev/null +++ b/tests/events/test_py_protocol.py @@ -0,0 +1,82 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + +from unittest.mock import Mock + +from synapse.api.room_versions import RoomVersion, RoomVersions +from synapse.events import EventBase, FrozenEvent, make_event_from_dict +from synapse.events.py_protocol import ( + EventProtocol, + MSC4242Event, + all_supports_msc4242_state_dag, + supports_msc4242_state_dag, +) + +from tests.unittest import TestCase + + +def _make_event(room_version: RoomVersion) -> EventBase: + """Helper to make an EventBase with the given room version.""" + event_dict = { + "content": {}, + "sender": "@user:example.com", + "type": "m.room.message", + "room_id": "!room:example.com", + } + if room_version.msc4242_state_dags: + event_dict["prev_state_events"] = [] + return make_event_from_dict(event_dict, room_version=room_version) + + +class TestMetaClass(TestCase): + def test_is_instance(self) -> None: + """Test that isinstance checks on EventProtocol raise + NotImplementedError, but that isinstance checks on EventBase and + FrozenEvent still work as normal. + """ + # EventBase and FrozenEvent should work as normal + self.assertFalse(isinstance(object(), EventBase)) + self.assertFalse(isinstance(object(), FrozenEvent)) + + with self.assertRaises(NotImplementedError): + isinstance(object(), EventProtocol) + + with self.assertRaises(NotImplementedError): + isinstance(object(), MSC4242Event) + + +class SupportsMSC4242StateDagTestCase(TestCase): + def test_single_event_msc4242(self) -> None: + """A single event in an MSC4242 room is recognised.""" + ev = _make_event(RoomVersions.MSC4242v12) + self.assertTrue(supports_msc4242_state_dag(ev)) + + def test_single_event_non_msc4242(self) -> None: + """A single event in a non-MSC4242 room is not recognised.""" + ev = _make_event(RoomVersions.V11) + self.assertFalse(supports_msc4242_state_dag(ev)) + + def test_sequence_all_msc4242(self) -> None: + """A sequence of MSC4242 (event, context) pairs is recognised.""" + pairs = [(_make_event(RoomVersions.MSC4242v12), Mock()) for _ in range(3)] + self.assertTrue(all_supports_msc4242_state_dag(pairs)) + + def test_sequence_mixed(self) -> None: + """A sequence containing any non-MSC4242 event is not recognised.""" + pairs = [ + (_make_event(RoomVersions.MSC4242v12), Mock()), + (_make_event(RoomVersions.V11), Mock()), + ] + self.assertFalse(all_supports_msc4242_state_dag(pairs)) diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index a402dd70d1..8f78ae4944 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -26,7 +26,7 @@ from synapse.api.constants import EventContentFields from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.events.utils import ( FilteredEvent, PowerLevelsContent, @@ -42,31 +42,16 @@ from synapse.types import JsonDict, create_requester from synapse.util.frozenutils import freeze +from tests.test_utils.event_builders import make_test_event from tests.unittest import HomeserverTestCase if TYPE_CHECKING: from synapse.server import HomeServer -def MockEvent(**kwargs: Any) -> EventBase: - if "event_id" not in kwargs: - kwargs["event_id"] = "fake_event_id" - if "type" not in kwargs: - kwargs["type"] = "fake_type" - if "content" not in kwargs: - kwargs["content"] = {} - - # Move internal metadata out so we can call make_event properly - internal_metadata = kwargs.get("internal_metadata") - if internal_metadata is not None: - kwargs.pop("internal_metadata") - - return make_event_from_dict(kwargs, internal_metadata_dict=internal_metadata) - - class TestMaybeUpsertEventField(stdlib_unittest.TestCase): def test_update_okay(self) -> None: - event = make_event_from_dict({"event_id": "$1234"}) + event = make_test_event({"event_id": "$1234"}) success = maybe_upsert_event_field( event, event.unsigned, "replaces_state", "value" ) @@ -74,7 +59,7 @@ def test_update_okay(self) -> None: self.assertEqual(event.unsigned["replaces_state"], "value") def test_update_not_okay(self) -> None: - event = make_event_from_dict({"event_id": "$1234"}) + event = make_test_event({"event_id": "$1234"}) LARGE_STRING = "a" * 100_000 success = maybe_upsert_event_field( event, event.unsigned, "replaces_state", LARGE_STRING @@ -83,7 +68,7 @@ def test_update_not_okay(self) -> None: self.assertNotIn("replaces_state", event.unsigned) def test_update_not_okay_leaves_original_value(self) -> None: - event = make_event_from_dict( + event = make_test_event( {"event_id": "$1234", "unsigned": {"replaces_state": "value"}} ) LARGE_STRING = "a" * 100_000 @@ -95,6 +80,20 @@ def test_update_not_okay_leaves_original_value(self) -> None: class PruneEventTestCase(stdlib_unittest.TestCase): + # Fields that `make_test_event` fills in by default and that `prune_event` + # preserves as spec-required keep fields. Pruning tests only spell out the + # fields they care about; these are merged into the expected dict so each + # test stays focused on what it is actually checking. + _DEFAULT_KEPT_FIELDS: JsonDict = { + "sender": "@test:test", + "room_id": "!test:test", + "depth": 1, + "origin_server_ts": 1, + "hashes": {}, + "auth_events": [], + "prev_events": [], + } + def run_test(self, evdict: JsonDict, matchdict: JsonDict, **kwargs: Any) -> None: """ Asserts that a new event constructed with `evdict` will look like @@ -105,8 +104,9 @@ def run_test(self, evdict: JsonDict, matchdict: JsonDict, **kwargs: Any) -> None matchdict: The expected resulting dictionary. kwargs: Additional keyword arguments used to create the event. """ + expected = {**self._DEFAULT_KEPT_FIELDS, **matchdict} self.assertEqual( - prune_event(make_event_from_dict(evdict, **kwargs)).get_dict(), matchdict + prune_event(make_test_event(evdict, **kwargs)).get_dict(), expected ) def test_minimal(self) -> None: @@ -123,9 +123,6 @@ def test_minimal(self) -> None: def test_basic_keys(self) -> None: """Ensure that the keys that should be untouched are kept.""" - # Note that some of the values below don't really make sense, but the - # pruning of events doesn't worry about the values of any fields (with - # the exception of the content field). self.run_test( { "event_id": "$3:domain", @@ -134,12 +131,12 @@ def test_basic_keys(self) -> None: "sender": "@2:domain", "state_key": "B", "content": {"other_key": "foo"}, - "hashes": "hashes", + "hashes": {"sha256": "abc"}, "signatures": {"domain": {"algo:1": "sigs"}}, "depth": 4, - "prev_events": "prev_events", + "prev_events": [], "prev_state": "prev_state", - "auth_events": "auth_events", + "auth_events": [], "origin": "domain", # historical top-level field that still exists on old events "origin_server_ts": 1234, "membership": "join", @@ -152,11 +149,11 @@ def test_basic_keys(self) -> None: "room_id": "!1:domain", "sender": "@2:domain", "state_key": "B", - "hashes": "hashes", + "hashes": {"sha256": "abc"}, "depth": 4, - "prev_events": "prev_events", + "prev_events": [], "prev_state": "prev_state", - "auth_events": "auth_events", + "auth_events": [], "origin": "domain", # historical top-level field that still exists on old events "origin_server_ts": 1234, "membership": "join", @@ -625,7 +622,7 @@ def test_relations(self) -> None: class CloneEventTestCase(stdlib_unittest.TestCase): def test_unsigned_is_copied(self) -> None: - original = make_event_from_dict( + original = make_test_event( { "type": "A", "event_id": "$test:domain", @@ -679,7 +676,8 @@ def serialize( def test_event_fields_works_with_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent(sender="@alice:localhost", room_id="!foo:bar"), ["room_id"] + make_test_event(sender="@alice:localhost", room_id="!foo:bar"), + ["room_id"], ), {"room_id": "!foo:bar"}, ) @@ -687,7 +685,7 @@ def test_event_fields_works_with_keys(self) -> None: def test_event_fields_works_with_nested_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={"body": "A message"}, @@ -700,7 +698,7 @@ def test_event_fields_works_with_nested_keys(self) -> None: def test_event_fields_works_with_dot_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={"key.with.dots": {}}, @@ -713,7 +711,7 @@ def test_event_fields_works_with_dot_keys(self) -> None: def test_event_fields_works_with_nested_dot_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={ @@ -729,7 +727,7 @@ def test_event_fields_works_with_nested_dot_keys(self) -> None: def test_event_fields_nops_with_unknown_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={"foo": "bar"}, @@ -742,7 +740,7 @@ def test_event_fields_nops_with_unknown_keys(self) -> None: def test_event_fields_nops_with_non_dict_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={"foo": ["I", "am", "an", "array"]}, @@ -755,7 +753,7 @@ def test_event_fields_nops_with_non_dict_keys(self) -> None: def test_event_fields_nops_with_array_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={"foo": ["I", "am", "an", "array"]}, @@ -768,7 +766,7 @@ def test_event_fields_nops_with_array_keys(self) -> None: def test_event_fields_all_fields_if_empty(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( type="foo", event_id="test", room_id="!foo:bar", @@ -782,6 +780,9 @@ def test_event_fields_all_fields_if_empty(self) -> None: "room_id": "!foo:bar", "content": {"foo": "bar"}, "unsigned": {}, + "sender": "@test:test", + "user_id": "@test:test", + "origin_server_ts": 1, }, ) @@ -799,12 +800,12 @@ def test_event_flagged_for_admins(self) -> None: # Default behaviour should be *not* to include it self.assertEqual( self.serialize( - MockEvent( + make_test_event( type="foo", event_id="test", room_id="!foo:bar", content={"foo": "bar"}, - internal_metadata={"soft_failed": True}, + internal_metadata_dict={"soft_failed": True}, ), [], ), @@ -814,18 +815,21 @@ def test_event_flagged_for_admins(self) -> None: "room_id": "!foo:bar", "content": {"foo": "bar"}, "unsigned": {}, + "sender": "@test:test", + "user_id": "@test:test", + "origin_server_ts": 1, }, ) # When asked though, we should set it self.assertEqual( self.serialize( - MockEvent( + make_test_event( type="foo", event_id="test", room_id="!foo:bar", content={"foo": "bar"}, - internal_metadata={"soft_failed": True}, + internal_metadata_dict={"soft_failed": True}, ), [], True, @@ -836,16 +840,19 @@ def test_event_flagged_for_admins(self) -> None: "room_id": "!foo:bar", "content": {"foo": "bar"}, "unsigned": {"io.element.synapse.soft_failed": True}, + "sender": "@test:test", + "user_id": "@test:test", + "origin_server_ts": 1, }, ) self.assertEqual( self.serialize( - MockEvent( + make_test_event( type="foo", event_id="test", room_id="!foo:bar", content={"foo": "bar"}, - internal_metadata={ + internal_metadata_dict={ "soft_failed": True, "policy_server_spammy": True, }, @@ -862,6 +869,9 @@ def test_event_flagged_for_admins(self) -> None: "io.element.synapse.soft_failed": True, "io.element.synapse.policy_server_spammy": True, }, + "sender": "@test:test", + "user_id": "@test:test", + "origin_server_ts": 1, }, ) @@ -896,7 +906,7 @@ def test_redacted_because_is_filtered_out(self) -> None: redaction_id = "$redaction_event_id" - event = MockEvent( + event = make_test_event( type="foo", event_id="test", room_id="!foo:bar", @@ -904,7 +914,7 @@ def test_redacted_because_is_filtered_out(self) -> None: ) event.internal_metadata.redacted_by = redaction_id - redaction_event = MockEvent( + redaction_event = make_test_event( type="m.room.redaction", event_id=redaction_id, content={"redacts": "test"}, diff --git a/tests/events/test_validator.py b/tests/events/test_validator.py new file mode 100644 index 0000000000..3810fdb3da --- /dev/null +++ b/tests/events/test_validator.py @@ -0,0 +1,50 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +from synapse.api.room_versions import RoomVersions +from synapse.events import make_event_from_dict +from synapse.events.validator import EventValidator + +from tests.unittest import HomeserverTestCase + + +class EventValidatorTestCase(HomeserverTestCase): + def test_validate_new_with_mentions_succeeds_even_when_frozen(self) -> None: + """ + Test that `EventValidator.validate_new` accepts an event with valid `m.mentions` + content even when the event is frozen. + """ + event = make_event_from_dict( + { + "room_id": "!room:test", + "type": "m.room.message", + "sender": "@alice:example.com", + "content": { + "msgtype": "m.text", + "body": "@alice:example.com hello", + "m.mentions": {"user_ids": ["@alice:example.com"]}, + }, + "auth_events": [], + "prev_events": [], + "hashes": {"sha256": "aGVsbG8="}, + "signatures": {}, + "depth": 1, + "origin_server_ts": 1000, + }, + room_version=RoomVersions.V9, + ) + # Sanity check that the event is valid before freezing + EventValidator().validate_new(event, self.hs.config) + event.freeze() + # Event should still be valid after freezing + EventValidator().validate_new(event, self.hs.config) diff --git a/tests/federation/test_federation_base.py b/tests/federation/test_federation_base.py index 1bc1da1feb..8b8c65a0c5 100644 --- a/tests/federation/test_federation_base.py +++ b/tests/federation/test_federation_base.py @@ -36,6 +36,9 @@ def test_events_signed_by_banned_key_are_refused(self) -> None: "type": "m.room.message", "room_id": "!r:domain", "sender": f"@u:{self.hs.config.server.server_name}", + "auth_events": [], + "prev_events": [], + "depth": 1, "signatures": {}, "unsigned": {"age_ts": 1000000}, } diff --git a/tests/federation/test_federation_client.py b/tests/federation/test_federation_client.py index 0535aed107..656665cb42 100644 --- a/tests/federation/test_federation_client.py +++ b/tests/federation/test_federation_client.py @@ -73,6 +73,7 @@ def test_get_room_state(self) -> None: "content": {"creator": self.creator}, "prev_events": [], "auth_events": [], + "depth": 1, "origin_server_ts": 500, } ) @@ -85,6 +86,7 @@ def test_get_room_state(self) -> None: "content": {"membership": "join"}, "prev_events": [], "auth_events": [], + "depth": 2, "origin_server_ts": 600, } ) @@ -97,6 +99,7 @@ def test_get_room_state(self) -> None: "content": {}, "prev_events": [], "auth_events": [], + "depth": 3, "origin_server_ts": 700, } ) diff --git a/tests/federation/test_federation_out_of_band_membership.py b/tests/federation/test_federation_out_of_band_membership.py index a1ab72b7a1..1707081863 100644 --- a/tests/federation/test_federation_out_of_band_membership.py +++ b/tests/federation/test_federation_out_of_band_membership.py @@ -33,11 +33,8 @@ from synapse.api.constants import EventContentFields, EventTypes, Membership from synapse.api.room_versions import RoomVersion, RoomVersions -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.events.utils import strip_event -from synapse.federation.federation_base import ( - event_from_pdu_json, -) from synapse.federation.transport.client import SendJoinResponse from synapse.http.matrixfederationclient import ( ByteParser, @@ -53,6 +50,7 @@ from synapse.util.clock import Clock from tests import unittest +from tests.test_utils.event_builders import make_test_event, make_test_pdu_event from tests.utils import test_timeout logger = logging.getLogger(__name__) @@ -80,9 +78,7 @@ def required_state_json_to_state_map(required_state: Any) -> StateMap[EventBase] "Each event in `required_state` should have a string `state_key`" ) - state_map[(event_type, event_state_key)] = make_event_from_dict( - state_event_dict - ) + state_map[(event_type, event_state_key)] = make_test_event(state_event_dict) else: # Yell because we're in a test and this is unexpected raise AssertionError("`required_state` should be a list of event dicts") @@ -195,7 +191,7 @@ def _invite_local_user_to_remote_room_and_join(self) -> RemoteRoomJoinResult: remote_room_id = f"!remote-room:{self.OTHER_SERVER_NAME}" room_version = RoomVersions.V10 - room_create_event = make_event_from_dict( + room_create_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": remote_room_id, @@ -217,7 +213,7 @@ def _invite_local_user_to_remote_room_and_join(self) -> RemoteRoomJoinResult: room_version=room_version, ) - creator_membership_event = make_event_from_dict( + creator_membership_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": remote_room_id, @@ -235,7 +231,7 @@ def _invite_local_user_to_remote_room_and_join(self) -> RemoteRoomJoinResult: ) # From the remote homeserver, invite user1 on the local homserver - user1_invite_membership_event = make_event_from_dict( + user1_invite_membership_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": remote_room_id, @@ -297,7 +293,7 @@ def _invite_local_user_to_remote_room_and_join(self) -> RemoteRoomJoinResult: # Prevent tight-looping to allow the `test_timeout` to work time.sleep(0.1) - user1_join_membership_event_template = make_event_from_dict( + user1_join_membership_event_template = make_test_event( { "room_id": remote_room_id, "sender": local_user1_id, @@ -377,7 +373,7 @@ async def put_json( and parser is not None ): # As the remote server, we need to sign the event before sending it back - user1_join_membership_event_signed = make_event_from_dict( + user1_join_membership_event_signed = make_test_event( self.add_hashes_and_signatures_from_other_server(data), room_version=room_version, ) @@ -404,12 +400,12 @@ async def put_json( if path.startswith("/_matrix/federation/v1/send/") and data is not None: for pdu in data.get("pdus", []): - event = event_from_pdu_json(pdu, room_version) + event = make_test_pdu_event(pdu, room_version) collected_pdus_from_hs1_federation_send.add(event.event_id) # Just acknowledge everything hs1 is trying to send hs2 return { - event_from_pdu_json(pdu, room_version).event_id: {} + make_test_pdu_event(pdu, room_version).event_id: {} for pdu in data.get("pdus", []) } @@ -521,12 +517,12 @@ async def put_json( ) -> JsonDict | T: if path.startswith("/_matrix/federation/v1/send/") and data is not None: for pdu in data.get("pdus", []): - event = event_from_pdu_json(pdu, room_version) + event = make_test_pdu_event(pdu, room_version) collected_pdus_from_hs1_federation_send.add(event.event_id) # Just acknowledge everything hs1 is trying to send hs2 return { - event_from_pdu_json(pdu, room_version).event_id: {} + make_test_pdu_event(pdu, room_version).event_id: {} for pdu in data.get("pdus", []) } @@ -538,7 +534,7 @@ async def put_json( self.federation_http_client.put_json.side_effect = put_json # From the remote homeserver, invite user2 on the local homserver - user2_invite_membership_event = make_event_from_dict( + user2_invite_membership_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": remote_room_id, diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index eef3c50cd0..22d5b0c094 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -31,8 +31,7 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config.server import DEFAULT_ROOM_VERSION from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import EventBase, make_event_from_dict -from synapse.federation.federation_base import event_from_pdu_json +from synapse.events import EventBase from synapse.http.types import QueryParams from synapse.logging.context import LoggingContext from synapse.rest import admin @@ -43,6 +42,7 @@ from synapse.util.clock import Clock from tests import unittest +from tests.test_utils.event_builders import make_test_event, make_test_pdu_event from tests.unittest import override_config logger = logging.getLogger(__name__) @@ -95,7 +95,7 @@ async def failing_handler(_origin: str, _content: JsonDict) -> None: def _create_acl_event(content: JsonDict) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": "!a:b", "event_id": "$a:b", @@ -147,7 +147,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: # Join a remote user to the room that will attempt to send bad events self.remote_bad_user_id = f"@baduser:{self.OTHER_SERVER_NAME}" - self.remote_bad_user_join_event = make_event_from_dict( + self.remote_bad_user_join_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": self.room_id, @@ -212,7 +212,7 @@ async def post_json( ) # Now lie about an event's prev_events - lying_event = make_event_from_dict( + lying_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": self.room_id, @@ -792,7 +792,7 @@ def test_strip_unauthorized_unsigned_values(self) -> None: "auth_events": [], "unsigned": {"malicious garbage": "hackz", "more warez": "more hackz"}, } - filtered_event = event_from_pdu_json(event1, RoomVersions.V1) + filtered_event = make_test_pdu_event(event1, RoomVersions.V1) # Make sure unauthorized fields are stripped from unsigned self.assertNotIn("more warez", filtered_event.unsigned) @@ -814,7 +814,7 @@ def test_strip_event_maintains_allowed_fields(self) -> None: }, } - filtered_event2 = event_from_pdu_json(event2, RoomVersions.V1, received_time=20) + filtered_event2 = make_test_pdu_event(event2, RoomVersions.V1, received_time=20) self.assertIn("age_ts", filtered_event2.unsigned) self.assertEqual(6, filtered_event2.unsigned["age_ts"]) self.assertNotIn("more warez", filtered_event2.unsigned) @@ -839,7 +839,7 @@ def test_strip_event_removes_fields_based_on_event_type(self) -> None: "invite_room_state": [], }, } - filtered_event3 = event_from_pdu_json(event3, RoomVersions.V1, received_time=20) + filtered_event3 = make_test_pdu_event(event3, RoomVersions.V1, received_time=20) self.assertIn("age_ts", filtered_event3.unsigned) # Invite_room_state field is only permitted in event type m.room.member self.assertNotIn("invite_room_state", filtered_event3.unsigned) diff --git a/tests/federation/transport/test_client.py b/tests/federation/transport/test_client.py index 9a6bbabd35..881baa02a9 100644 --- a/tests/federation/transport/test_client.py +++ b/tests/federation/transport/test_client.py @@ -36,7 +36,16 @@ class SendJoinParserTestCase(TestCase): def test_two_writes(self) -> None: """Test that the parser can sensibly deserialise an input given in two slices.""" parser = SendJoinParser(RoomVersions.V1, True) + common_event_fields = { + "sender": "@user:example.org", + "depth": 1, + "origin_server_ts": 1, + "hashes": {}, + "auth_events": [], + "prev_events": [], + } parent_event = { + **common_event_fields, "content": { "see_room_version_spec": "The event format changes depending on the room version." }, @@ -45,6 +54,7 @@ def test_two_writes(self) -> None: "type": "m.room.minimal_pdu", } state = { + **common_event_fields, "content": { "see_room_version_spec": "The event format changes depending on the room version." }, diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 20ffed68f4..794c0a3185 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -35,8 +35,7 @@ SynapseError, ) from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, make_event_from_dict -from synapse.federation.federation_base import event_from_pdu_json +from synapse.events import EventBase from synapse.federation.federation_client import SendJoinResult from synapse.rest import admin from synapse.rest.client import login, room @@ -45,6 +44,7 @@ from synapse.util.clock import Clock from tests import unittest +from tests.test_utils.event_builders import make_test_event, make_test_pdu_event logger = logging.getLogger(__name__) @@ -132,7 +132,7 @@ def test_rejected_message_event_state(self) -> None: ) # build and send an event which will be rejected - ev = event_from_pdu_json( + ev = make_test_pdu_event( { "type": EventTypes.Message, "content": {}, @@ -183,7 +183,7 @@ def test_rejected_state_event_state(self) -> None: ) # build and send an event which will be rejected - ev = event_from_pdu_json( + ev = make_test_pdu_event( { "type": "org.matrix.test", "state_key": "test_key", @@ -227,7 +227,7 @@ def test_backfill_ignores_known_events(self) -> None: room_version = self.get_success(self.store.get_room_version(room_id)) # Build an event to backfill - event = event_from_pdu_json( + event = make_test_pdu_event( { "type": EventTypes.Message, "content": {"body": "hello world", "msgtype": "m.text"}, @@ -324,7 +324,7 @@ def test_invite_by_user_ratelimit(self) -> None: def create_invite() -> EventBase: room_id = self.helper.create_room_as(room_creator=user_id, tok=tok) room_version = self.get_success(self.store.get_room_version(room_id)) - return event_from_pdu_json( + return make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, @@ -386,7 +386,7 @@ def _build_and_send_join_event( class EventFromPduTestCase(TestCase): def test_valid_json(self) -> None: """Valid JSON should be turned into an event.""" - ev = event_from_pdu_json( + ev = make_test_pdu_event( { "type": EventTypes.Message, "content": {"bool": True, "null": None, "int": 1, "str": "foobar"}, @@ -413,7 +413,7 @@ def test_invalid_numbers(self) -> None: float("nan"), ]: with self.assertRaises(SynapseError): - event_from_pdu_json( + make_test_pdu_event( { "type": EventTypes.Message, "content": {"foo": value}, @@ -430,7 +430,7 @@ def test_invalid_numbers(self) -> None: def test_invalid_nested(self) -> None: """List and dictionaries are recursively searched.""" with self.assertRaises(SynapseError): - event_from_pdu_json( + make_test_pdu_event( { "type": EventTypes.Message, "content": {"foo": [{"bar": 2**56}]}, @@ -457,7 +457,7 @@ def test_failed_partial_join_is_clean(self) -> None: room_id = "!room:example.com" - EVENT_CREATE = make_event_from_dict( + EVENT_CREATE = make_test_event( { "room_id": room_id, "type": "m.room.create", @@ -470,7 +470,7 @@ def test_failed_partial_join_is_clean(self) -> None: }, room_version=RoomVersions.V10, ) - EVENT_CREATOR_MEMBERSHIP = make_event_from_dict( + EVENT_CREATOR_MEMBERSHIP = make_test_event( { "room_id": room_id, "type": "m.room.member", @@ -484,7 +484,7 @@ def test_failed_partial_join_is_clean(self) -> None: }, room_version=RoomVersions.V10, ) - EVENT_INVITATION_MEMBERSHIP = make_event_from_dict( + EVENT_INVITATION_MEMBERSHIP = make_test_event( { "room_id": room_id, "type": "m.room.member", @@ -501,7 +501,7 @@ def test_failed_partial_join_is_clean(self) -> None: }, room_version=RoomVersions.V10, ) - membership_event = make_event_from_dict( + membership_event = make_test_event( { "room_id": room_id, "type": "m.room.member", diff --git a/tests/handlers/test_federation_event.py b/tests/handlers/test_federation_event.py index 1aaa86e2e8..9da9f52fb6 100644 --- a/tests/handlers/test_federation_event.py +++ b/tests/handlers/test_federation_event.py @@ -28,7 +28,6 @@ check_state_dependent_auth_rules, check_state_independent_auth_rules, ) -from synapse.events import make_event_from_dict from synapse.events.snapshot import EventContext from synapse.federation.transport.client import StateRequestResponse from synapse.logging.context import LoggingContext @@ -42,6 +41,7 @@ from tests import unittest from tests.test_utils import event_injection +from tests.test_utils.event_builders import make_test_event class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): @@ -127,7 +127,7 @@ def _test_process_pulled_event_with_missing_state( # mock up a load of state events which we are missing state_events = [ - make_event_from_dict( + make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_state_type", @@ -154,7 +154,7 @@ def _test_process_pulled_event_with_missing_state( # mock up a prev event. # Depending on the test, we either persist this upfront (as an outlier), # or let the server request it. - prev_event = make_event_from_dict( + prev_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -191,7 +191,7 @@ async def get_event( self.mock_federation_transport_client.get_event.side_effect = get_event # mock up a regular event to pass into _process_pulled_event - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -301,7 +301,7 @@ def test_process_pulled_event_records_failed_backfill_attempts( ) ) - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -421,7 +421,7 @@ def test_process_pulled_event_clears_backfill_attempts_after_being_successfully_ member_event.event_id, ] - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -524,7 +524,7 @@ def test_backfill_signature_failure_does_not_fetch_same_prev_event_later( # We purposely don't run `add_hashes_and_signatures_from_other_server` # over this because we want the signature check to fail. - pulled_event_without_signatures = make_event_from_dict( + pulled_event_without_signatures = make_test_event( { "type": "test_regular_type", "room_id": room_id, @@ -540,7 +540,7 @@ def test_backfill_signature_failure_does_not_fetch_same_prev_event_later( # Create a regular event that should pass except for the # `pulled_event_without_signatures` in the `prev_event`. - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -720,7 +720,7 @@ def test_backfill_process_previously_failed_pull_attempt_event_in_the_background ] # Create a regular event that should process - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -878,7 +878,7 @@ def test_process_pulled_event_with_rejected_missing_state(self) -> None: # accepted, but the local homeserver will reject. next_depth = 100 next_timestamp = other_member_event.origin_server_ts + 100 - rejected_power_levels_event = make_event_from_dict( + rejected_power_levels_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "m.room.power_levels", @@ -927,7 +927,7 @@ def test_process_pulled_event_with_rejected_missing_state(self) -> None: # Then we create a kick event for a local user that cites the rejected power # levels event in its auth events. The kick event will be rejected solely # because of the rejected auth event and would otherwise be accepted. - rejected_kick_event = make_event_from_dict( + rejected_kick_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "m.room.member", @@ -1042,7 +1042,7 @@ def test_process_pulled_event_with_rejected_missing_state(self) -> None: # Create a missing event, so that the local homeserver has to do a `/state` or # `/state_ids` request to pull state from the remote homeserver. - missing_event = make_event_from_dict( + missing_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "m.room.message", @@ -1067,7 +1067,7 @@ def test_process_pulled_event_with_rejected_missing_state(self) -> None: # The pulled event has two prev events, one of which is missing. We will make a # `/state` or `/state_ids` request to the remote homeserver to ask it for the # state before the missing prev event. - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "m.room.message", diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index 3890abdbc8..b550c2420b 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -9,9 +9,6 @@ from synapse.api.errors import Codes, LimitExceededError, SynapseError from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.events import FrozenEventV3 -from synapse.federation.federation_base import ( - event_from_pdu_json, -) from synapse.federation.federation_client import SendJoinResult from synapse.server import HomeServer from synapse.types import UserID, create_requester @@ -19,6 +16,7 @@ from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.server import make_request +from tests.test_utils.event_builders import make_test_pdu_event from tests.unittest import ( FederatingHomeserverTestCase, HomeserverTestCase, @@ -549,7 +547,7 @@ def test_msc4155_block_invite_remote(self) -> None: ) room_version = self.get_success(self.store.get_room_version(room_id)) - invite_event = event_from_pdu_json( + invite_event = make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, @@ -595,7 +593,7 @@ def test_msc4155_block_invite_remote_server(self) -> None: ) room_version = self.get_success(self.store.get_room_version(room_id)) - invite_event = event_from_pdu_json( + invite_event = make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, @@ -710,7 +708,7 @@ def test_msc4380_block_invite_remote(self) -> None: ) room_version = self.get_success(self.store.get_room_version(room_id)) - invite_event = event_from_pdu_json( + invite_event = make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, diff --git a/tests/handlers/test_room_policy.py b/tests/handlers/test_room_policy.py index 9adf0e38c2..4f2188b8e7 100644 --- a/tests/handlers/test_room_policy.py +++ b/tests/handlers/test_room_policy.py @@ -22,16 +22,18 @@ from synapse.api.constants import EventTypes from synapse.api.errors import HttpResponseException, SynapseError from synapse.crypto.event_signing import compute_event_signature -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.handlers.room_policy import POLICY_SERVER_KEY_ID from synapse.rest import admin from synapse.rest.client import filter, login, room, sync from synapse.server import HomeServer +from synapse.synapse_rust.events import Signatures from synapse.types import JsonDict, UserID from synapse.util.clock import Clock from tests import unittest from tests.test_utils import event_injection +from tests.test_utils.event_builders import make_test_event class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): @@ -75,7 +77,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.signing_key = signedjson.key.generate_signing_key("policy_server") # Create some sample events - self.spammy_event = make_event_from_dict( + self.spammy_event = make_test_event( room_version=room_version, internal_metadata_dict={}, event_dict={ @@ -88,7 +90,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: }, }, ) - self.not_spammy_event = make_event_from_dict( + self.not_spammy_event = make_test_event( room_version=room_version, internal_metadata_dict={}, event_dict={ @@ -112,7 +114,15 @@ async def policy_server_signs_event( self.OTHER_SERVER_NAME, self.signing_key, ) - return sigs + # Only return the new signature like the policy server spec says, + # not any others that were already in the event + return { + self.OTHER_SERVER_NAME: { + POLICY_SERVER_KEY_ID: sigs[self.OTHER_SERVER_NAME][ + POLICY_SERVER_KEY_ID + ] + } + } async def policy_server_signs_event_with_wrong_key( destination: str, pdu: EventBase, timeout: int | None = None @@ -168,6 +178,19 @@ def _add_policy_server_to_room(self, public_key: str | None = None) -> None: state_key="", ) + def _sign_with_random_key(self, server_name: str, event: EventBase) -> None: + non_policyserver_key = signedjson.key.generate_signing_key( + "non_policyserver_key" + ) + event.signatures = Signatures( + compute_event_signature( + event.room_version, + event.get_dict(), + server_name, + non_policyserver_key, + ) + ) + def test_no_policy_event_set(self) -> None: # We don't need to modify the room state at all - we're testing the default # case where a room doesn't use a policy server. @@ -272,7 +295,7 @@ def test_spammy_event_is_spam(self) -> None: def test_signed_event_is_not_spam(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) self._add_policy_server_to_room(public_key=verify_key_str) - event = make_event_from_dict( + event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ @@ -302,7 +325,7 @@ def test_signed_event_is_not_spam(self) -> None: def test_ask_policy_server_to_sign_event_ok(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) self._add_policy_server_to_room(public_key=verify_key_str) - event = make_event_from_dict( + event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ @@ -315,16 +338,74 @@ def test_ask_policy_server_to_sign_event_ok(self) -> None: }, }, ) + # Sign the event as the origin server first, since that's what events passed to + # ask_policy_server_to_sign_event will generally look like. The exact key used + # here isn't important. + self._sign_with_random_key("example.org", event) + self.mock_federation_transport_client.ask_policy_server_to_sign_event.side_effect = self.policy_server_signs_event + self.get_success( + self.handler.ask_policy_server_to_sign_event(event, verify=True) + ) + # Standard success case: event has signatures from the origin and the policy server + self.assertEqual( + { + server: len(signatures) + for server, signatures in event.signatures.as_dict().items() + }, + {"example.org": 1, self.OTHER_SERVER_NAME: 1}, + f"Expected signatures for the origin homeserver (example.org) and policy server ({self.OTHER_SERVER_NAME})", + ) + + def test_ask_origin_server_to_sign_event_doesnt_replace_signatures(self) -> None: + """ + ``ask_policy_server_to_sign_event`` has had bugs where it accidentally overwrote + the origin server's signature in the case where the origin server has the same + server name as the policy server (each have their own signing key). This test is + otherwise equivalent to the success case test above, but the server name for + origin event sending server and the policy server are the same and we want to + ensure both signatures are preserved. + """ + verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) + self._add_policy_server_to_room(public_key=verify_key_str) + event = make_test_event( + room_version=self.room_version, + internal_metadata_dict={}, + event_dict={ + "room_id": self.room_id, + "type": "m.room.message", + "sender": "@spammy:" + self.OTHER_SERVER_NAME, + "content": { + "msgtype": "m.text", + "body": "This is another signed event.", + }, + }, + ) + # Sign the event as the origin server that sent the event, which in this case + # has the same server name as the policy server. We're using a different key + # than `self.signing_key` (for the policy server), as the ed25519:policy_server + # key is only used for policy server signatures, not any other federation traffic + # even when the origin server and policy are logically the same server. + self._sign_with_random_key(self.OTHER_SERVER_NAME, event) self.mock_federation_transport_client.ask_policy_server_to_sign_event.side_effect = self.policy_server_signs_event self.get_success( self.handler.ask_policy_server_to_sign_event(event, verify=True) ) - self.assertEqual(len(event.signatures), 1) + # Less common success case: the event origin server is logically the same as + # the policy server, so there will be two signatures from one server name. + # It's important to make sure both signatures are preserved. + self.assertEqual( + { + server: len(signatures) + for server, signatures in event.signatures.as_dict().items() + }, + {self.OTHER_SERVER_NAME: 2}, + f"Expected 2 signatures for the origin server and policy server under the same server name ({self.OTHER_SERVER_NAME}) but with different keys", + ) def test_ask_policy_server_to_sign_event_refuses(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) self._add_policy_server_to_room(public_key=verify_key_str) - event = make_event_from_dict( + event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ @@ -353,7 +434,7 @@ def test_ask_policy_server_to_sign_event_refuses(self) -> None: def test_ask_policy_server_to_sign_event_cannot_reach(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) self._add_policy_server_to_room(public_key=verify_key_str) - event = make_event_from_dict( + event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ @@ -379,7 +460,7 @@ def test_ask_policy_server_to_sign_event_wrong_sig(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) self._add_policy_server_to_room(public_key=verify_key_str) self.mock_federation_transport_client.ask_policy_server_to_sign_event.side_effect = self.policy_server_signs_event_with_wrong_key - unverified_event = make_event_from_dict( + unverified_event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ @@ -398,7 +479,7 @@ def test_ask_policy_server_to_sign_event_wrong_sig(self) -> None: ) self.assertEqual(len(unverified_event.signatures), 1) - verified_event = make_event_from_dict( + verified_event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ diff --git a/tests/handlers/test_room_summary.py b/tests/handlers/test_room_summary.py index ee65cb1afb..49076e69d8 100644 --- a/tests/handlers/test_room_summary.py +++ b/tests/handlers/test_room_summary.py @@ -35,7 +35,6 @@ ) from synapse.api.errors import AuthError, NotFoundError, SynapseError from synapse.api.room_versions import RoomVersions -from synapse.events import make_event_from_dict from synapse.federation.transport.client import TransportLayerClient from synapse.handlers.room_summary import _child_events_comparison_key, _RoomEntry from synapse.rest import admin @@ -45,6 +44,7 @@ from synapse.util.clock import Clock from tests import unittest +from tests.test_utils.event_builders import make_test_event from tests.unittest import override_config @@ -217,7 +217,7 @@ def _poke_fed_invite(self, room_id: str, from_user: str) -> None: # Poke an invite over federation into the database. fed_handler = self.hs.get_federation_handler() fed_hostname = UserID.from_string(from_user).domain - event = make_event_from_dict( + event = make_test_event( { "room_id": room_id, "event_id": "!abcd:" + fed_hostname, diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index b9dee1c954..d2b2523321 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -32,7 +32,6 @@ from synapse.api.room_versions import RoomVersion, RoomVersions from synapse.events import EventBase from synapse.events.snapshot import EventContext -from synapse.federation.federation_base import event_from_pdu_json from synapse.handlers.sync import ( SyncConfig, SyncRequestKey, @@ -51,9 +50,11 @@ create_requester, ) from synapse.util.clock import Clock +from synapse.util.duration import Duration import tests.unittest import tests.utils +from tests.test_utils.event_builders import make_test_pdu_event _request_key = 0 @@ -912,7 +913,7 @@ async def _check_sigs_and_hash_for_pulled_events_and_fetch( prev_events = self.get_success(self.store.get_prev_events_for_room(room_id)) # create a call invite event - call_event = event_from_pdu_json( + call_event = make_test_pdu_event( { "type": EventTypes.CallInvite, "content": {}, @@ -960,7 +961,7 @@ async def _check_sigs_and_hash_for_pulled_events_and_fetch( priv_prev_events = self.get_success( self.store.get_prev_events_for_room(private_room_id) ) - private_call_event = event_from_pdu_json( + private_call_event = make_test_pdu_event( { "type": EventTypes.CallInvite, "content": {}, @@ -1058,13 +1059,22 @@ def test_wait_for_future_sync_token(self) -> None: ) # This should block waiting for the presence stream to update - self.pump() + # + # Advance time a little bit to make the + # `wait_for_stream_token(...)` sleep loop iterate. + self.reactor.advance(Duration(seconds=2).as_secs()) + # It should still not be done yet self.assertFalse(sync_d.called) # Marking the stream ID as persisted should unblock the request. self.get_success(ctx_mgr.__aexit__(None, None, None)) - self.get_success(sync_d, by=1.0) + # Advance time to make another iteration of `wait_for_stream_token(...)` sleep + # loop so it sees that we're finally caught up now. + self.reactor.advance(Duration(seconds=1).as_secs()) + + # Done waiting + self.get_success(sync_d) @parameterized.expand( [(key,) for key in StreamKeyType.__members__.values()], diff --git a/tests/handlers/test_worker_lock.py b/tests/handlers/test_worker_lock.py index e4b3e9b2ce..a28d8e34d3 100644 --- a/tests/handlers/test_worker_lock.py +++ b/tests/handlers/test_worker_lock.py @@ -25,8 +25,13 @@ from twisted.internet import defer from twisted.internet.testing import MemoryReactor +from synapse.handlers.worker_lock import WORKER_LOCK_MAX_RETRY_INTERVAL from synapse.server import HomeServer -from synapse.storage.databases.main.lock import _RENEWAL_INTERVAL +from synapse.storage.databases.main.lock import ( + _LOCK_REAP_INTERVAL, + _LOCK_TIMEOUT, + _RENEWAL_INTERVAL, +) from synapse.util.clock import Clock from synapse.util.duration import Duration @@ -84,7 +89,7 @@ def test_timeouts_for_lock_locally(self) -> None: # Note: We use `_pump_by` instead of `pump`/`advance` as the `Lock` has an # internal background looping call that runs every 30 seconds # (`_RENEWAL_INTERVAL`) to renew the `Lock` and push it's "drop timeout" value - # further out by 2 minutes (`_LOCK_TIMEOUT_MS`). The `Lock` will prematurely + # further out by 2 minutes (`_LOCK_TIMEOUT`). The `Lock` will prematurely # drop if this renewal is not allowed to run, which sours the test. # self.pump(amount=Duration(hours=1)) self._pump_by(amount=Duration(hours=1), by=_RENEWAL_INTERVAL) @@ -92,9 +97,34 @@ def test_timeouts_for_lock_locally(self) -> None: # Make sure we haven't acquired the `lock2` yet (`lock1` still holds it) self.assertNoResult(d2) - # Release the first lock (`lock1`). The second lock(`lock2`) should be - # automatically acquired by the `pump()` inside `get_success()` - self.get_success(lock1.__aexit__(None, None, None)) + # Drop the lock without releasing it. If we just normally released the lock + # (`self.get_success(lock1.__aexit__(None, None, None))`), the + # `add_lock_released_callback`/`notify_lock_released` cycle would signal that we + # should re-aquire the lock right away (on the next reactor tick). And we want + # to avoid that as the point of this test is to stress the retry timeout + # interval and `WORKER_LOCK_MAX_RETRY_INTERVAL`. + del lock1 + + # Wait for `lock1` to go stale (it won't be renewed anymore because we deleted + # it just above) + self._pump_by( + amount=_LOCK_TIMEOUT, + by=_RENEWAL_INTERVAL, + ) + + # Wait just enough time so `lock1` is reaped (found stale and forcefully drops + # the lock its holding) + self._pump_by( + amount=_LOCK_REAP_INTERVAL, + by=_RENEWAL_INTERVAL, + ) + + # Wait just enough time so `lock2` tries re-acquiring the lock. Should be no + # longer than our `WORKER_LOCK_MAX_RETRY_INTERVAL`. + self._pump_by( + amount=WORKER_LOCK_MAX_RETRY_INTERVAL, + by=_RENEWAL_INTERVAL, + ) # We should now have the lock self.successResultOf(d2) @@ -230,7 +260,7 @@ def test_timeouts_for_lock_worker(self) -> None: # Note: We use `_pump_by` instead of `pump`/`advance` as the `Lock` has an # internal background looping call that runs every 30 seconds # (`_RENEWAL_INTERVAL`) to renew the `Lock` and push it's "drop timeout" value - # further out by 2 minutes (`_LOCK_TIMEOUT_MS`). The `Lock` will prematurely + # further out by 2 minutes (`_LOCK_TIMEOUT`). The `Lock` will prematurely # drop if this renewal is not allowed to run, which sours the test. # self.pump(amount=Duration(hours=1)) self._pump_by(amount=Duration(hours=1), by=_RENEWAL_INTERVAL) @@ -238,9 +268,34 @@ def test_timeouts_for_lock_worker(self) -> None: # Make sure we haven't acquired the `lock2` yet (`lock1` still holds it) self.assertNoResult(d2) - # Release the first lock (`lock1`). The second lock(`lock2`) should be - # automatically acquired by the `pump()` inside `get_success()` - self.get_success(lock1.__aexit__(None, None, None)) + # Drop the lock without releasing it. If we just normally released the lock + # (`self.get_success(lock1.__aexit__(None, None, None))`), the + # `add_lock_released_callback`/`notify_lock_released` cycle would signal that we + # should re-aquire the lock right away (on the next reactor tick). And we want + # to avoid that as the point of this test is to stress the retry timeout + # interval and `WORKER_LOCK_MAX_RETRY_INTERVAL`. + del lock1 + + # Wait for `lock1` to go stale (it won't be renewed anymore because we deleted + # it just above) + self._pump_by( + amount=_LOCK_TIMEOUT, + by=_RENEWAL_INTERVAL, + ) + + # Wait just enough time so `lock1` is reaped (found stale and forcefully drops + # the lock its holding) + self._pump_by( + amount=_LOCK_REAP_INTERVAL, + by=_RENEWAL_INTERVAL, + ) + + # Wait just enough time so `lock2` tries re-acquiring the lock. Should be no + # longer than our `WORKER_LOCK_MAX_RETRY_INTERVAL`. + self._pump_by( + amount=WORKER_LOCK_MAX_RETRY_INTERVAL, + by=_RENEWAL_INTERVAL, + ) # We should now have the lock self.successResultOf(d2) diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py index f1b20a12ec..2ba5da3b95 100644 --- a/tests/module_api/test_api.py +++ b/tests/module_api/test_api.py @@ -265,7 +265,7 @@ def test_sending_events_into_room(self) -> None: self.assertEqual(event.type, "m.room.message") self.assertEqual(event.room_id, room_id) self.assertFalse(hasattr(event, "state_key")) - self.assertDictEqual(event.content, content) + self.assertDictEqual(dict(event.content), content) expected_requester = create_requester( user_id, authenticated_entity=self.hs.hostname @@ -301,7 +301,7 @@ def test_sending_events_into_room(self) -> None: self.assertEqual(event.type, "m.room.power_levels") self.assertEqual(event.room_id, room_id) self.assertEqual(event.state_key, "") - self.assertDictEqual(event.content, content) + self.assertDictEqual(dict(event.content), content) # Check that the event was sent self.event_creation_handler.create_and_send_nonmember_event.assert_called_with( diff --git a/tests/push/test_presentable_names.py b/tests/push/test_presentable_names.py index 2558f2c0b2..2d382c8fab 100644 --- a/tests/push/test_presentable_names.py +++ b/tests/push/test_presentable_names.py @@ -23,11 +23,12 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, FrozenEvent +from synapse.events import EventBase from synapse.push.presentable_names import calculate_room_name from synapse.types import StateKey, StateMap from tests import unittest +from tests.test_utils.event_builders import make_test_event class MockDataStore: @@ -44,7 +45,7 @@ def __init__(self, events: Iterable[tuple[StateKey, dict]]): self._events = {} for i, (event_id, content) in enumerate(events): - self._events[event_id] = FrozenEvent( + self._events[event_id] = make_test_event( { "event_id": "$event_id", "type": event_id[0], @@ -59,7 +60,7 @@ def __init__(self, events: Iterable[tuple[StateKey, dict]]): async def get_event( self, event_id: str, allow_none: bool = False - ) -> FrozenEvent | None: + ) -> EventBase | None: assert allow_none, "Mock not configured for allow_none = False" # Decode the state key from the event ID. diff --git a/tests/push/test_push_rule_evaluator.py b/tests/push/test_push_rule_evaluator.py index a786d74bf1..2e389710b9 100644 --- a/tests/push/test_push_rule_evaluator.py +++ b/tests/push/test_push_rule_evaluator.py @@ -27,7 +27,6 @@ from synapse.api.constants import EventTypes, HistoryVisibility, Membership from synapse.api.room_versions import RoomVersions from synapse.appservice import ApplicationService -from synapse.events import FrozenEvent, make_event_from_dict from synapse.push.bulk_push_rule_evaluator import _flatten_dict from synapse.push.httppusher import tweaks_for_actions from synapse.rest import admin @@ -40,6 +39,7 @@ from synapse.util.frozenutils import freeze from tests import unittest +from tests.test_utils.event_builders import make_test_event from tests.test_utils.event_injection import create_event, inject_member_event @@ -81,7 +81,7 @@ def test_non_string(self) -> None: def test_event(self) -> None: """Events can also be flattened.""" - event = make_event_from_dict( + event = make_test_event( { "room_id": "!test:test", "type": "m.room.message", @@ -103,6 +103,10 @@ def test_event(self) -> None: "room_id": "!test:test", "sender": "@alice:test", "type": "m.room.message", + "depth": 1, + "origin_server_ts": 1, + "auth_events": [], + "prev_events": [], } self.assertEqual(expected, _flatten_dict(event)) @@ -121,24 +125,32 @@ def test_extensible_events(self) -> None: } # For a current room version, there's no special behavior. - event = make_event_from_dict(event_dict, room_version=RoomVersions.V8) + event = make_test_event(event_dict, room_version=RoomVersions.V8) expected = { "room_id": "!test:test", "sender": "@alice:test", "type": "m.room.message", "content.org\\.matrix\\.msc1767\\.markup": [], + "depth": 1, + "origin_server_ts": 1, + "auth_events": [], + "prev_events": [], } self.assertEqual(expected, _flatten_dict(event)) # For a room version with extensible events, they parse out the text/plain # to a content.body property. - event = make_event_from_dict(event_dict, room_version=RoomVersions.MSC1767v10) + event = make_test_event(event_dict, room_version=RoomVersions.MSC1767v10) expected = { "content.body": "hello world!", "room_id": "!test:test", "sender": "@alice:test", "type": "m.room.message", "content.org\\.matrix\\.msc1767\\.markup": [], + "depth": 1, + "origin_server_ts": 1, + "auth_events": [], + "prev_events": [], } self.assertEqual(expected, _flatten_dict(event)) @@ -152,7 +164,7 @@ def _get_evaluator( msc4210: bool = False, msc4306: bool = False, ) -> PushRuleEvaluator: - event = FrozenEvent( + event = make_test_event( { "event_id": "$event_id", "type": "m.room.history_visibility", diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 507cf10c5d..c4e4170c6f 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -2549,7 +2549,7 @@ def test_timestamp_to_event(self) -> None: def test_topo_token_is_accepted(self) -> None: """Test Topo Token is accepted.""" - token = "t1-0_0_0_0_0_0_0_0_0_0_0_0" + token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), @@ -2563,7 +2563,7 @@ def test_topo_token_is_accepted(self) -> None: def test_stream_token_is_accepted_for_fwd_pagianation(self) -> None: """Test that stream token is accepted for forward pagination.""" - token = "s0_0_0_0_0_0_0_0_0_0_0_0" + token = "s0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), diff --git a/tests/rest/client/sliding_sync/test_rooms_meta.py b/tests/rest/client/sliding_sync/test_rooms_meta.py index 4ff07b4d26..b1b771ef84 100644 --- a/tests/rest/client/sliding_sync/test_rooms_meta.py +++ b/tests/rest/client/sliding_sync/test_rooms_meta.py @@ -631,6 +631,76 @@ def test_rooms_meta_heroes_max(self) -> None: # We didn't request any state so we shouldn't see any `required_state` self.assertIsNone(response_body["rooms"][room_id1].get("required_state")) + def test_rooms_meta_heroes_empty_room_name(self) -> None: + """ + Test that the `rooms` `heroes` are included when the room name is an + empty string (i.e. unset as per the spec) + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + user3_id = self.register_user("user3", "pass") + _user3_tok = self.login(user3_id, "pass") + + room_id = self.helper.create_room_as( + user2_id, + tok=user2_tok, + extra_content={ + # https://spec.matrix.org/v1.17/client-server-api/#mroomname + # > If a room has an m.room.name event with an absent, null, or + # > empty name field, it should be treated the same as a room + # > with no m.room.name event. + "name": "", + }, + ) + self.helper.join(room_id, user1_id, tok=user1_tok) + # User3 is invited + self.helper.invite(room_id, src=user2_id, targ=user3_id, tok=user2_tok) + + # Make the Sliding Sync request + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [], + "timeline_limit": 1, + } + } + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Room has an empty name so we should see `heroes` populated + self.assertEqual(response_body["rooms"][room_id]["initial"], True) + self.assertIsNone(response_body["rooms"][room_id].get("name")) + self.assertCountEqual( + [ + hero["user_id"] + for hero in response_body["rooms"][room_id].get("heroes", []) + ], + # Heroes shouldn't include the user themselves (we shouldn't see user1) + [user2_id, user3_id], + ) + self.assertEqual( + response_body["rooms"][room_id]["joined_count"], + 2, + ) + self.assertEqual( + response_body["rooms"][room_id]["invited_count"], + 1, + ) + + # We didn't request any state so we shouldn't see any `required_state` + self.assertIsNone(response_body["rooms"][room_id].get("required_state")) + + # Send a message to make the room come down sync + self.helper.send(room_id, "message in room", tok=user2_tok) + + # Incremental sync + incremental_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok) + self.assertNotIn("name", incremental_body["rooms"][room_id]) + self.assertNotIn("heroes", incremental_body["rooms"][room_id]) + def test_rooms_meta_heroes_when_banned(self) -> None: """ Test that the `rooms` `heroes` are included in the response when the room diff --git a/tests/rest/client/sliding_sync/test_sliding_sync.py b/tests/rest/client/sliding_sync/test_sliding_sync.py index ac8dfd37d8..ebf41cd87c 100644 --- a/tests/rest/client/sliding_sync/test_sliding_sync.py +++ b/tests/rest/client/sliding_sync/test_sliding_sync.py @@ -30,7 +30,7 @@ RoomTypes, ) from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, StrippedStateEvent, make_event_from_dict +from synapse.events import EventBase, StrippedStateEvent from synapse.events.snapshot import EventContext from synapse.handlers.sliding_sync import StateValues from synapse.rest.client import account_data, devices, login, receipts, room, sync @@ -47,6 +47,7 @@ from tests import unittest from tests.server import FakeChannel, TimedOutException +from tests.test_utils.event_builders import make_test_event from tests.test_utils.event_injection import create_event logger = logging.getLogger(__name__) @@ -308,7 +309,7 @@ def _create_remote_invite_room_for_user( "invite_room_state": serialized_stripped_state_events } - invite_event = make_event_from_dict( + invite_event = make_test_event( invite_event_dict, room_version=RoomVersions.V10, ) diff --git a/tests/rest/client/test_auth.py b/tests/rest/client/test_auth.py index e7cfae4693..c62d83d5bc 100644 --- a/tests/rest/client/test_auth.py +++ b/tests/rest/client/test_auth.py @@ -36,9 +36,7 @@ devices, login, logout, - read_marker, register, - room, ) from synapse.rest.synapse.client import build_synapse_client_resource_tree from synapse.server import HomeServer @@ -777,9 +775,7 @@ class RefreshAuthTests(unittest.HomeserverTestCase): devices.register_servlets, login.register_servlets, logout.register_servlets, - read_marker.register_servlets, register.register_servlets, - room.register_servlets, synapse.rest.admin.register_servlets_for_client_rest_resource, ] hijack_auth = False @@ -1548,14 +1544,16 @@ def _txn(txn: LoggingTransaction) -> int: def test_token_invalid_after_refresh_token_issued_and_device_removed(self) -> None: """ - Test that sending a read marker (dummy action) fails after the device (which had a refresh token) is removed by another device. - The removal of a refresh token cascade deletes the associated access token in the db, which can make cache invalidation fail, if not handled properly. - This test will catch such behavior if it ever happens again. + Test that an access token is invalidated after the device (which had a + refresh token) is removed by another device. + The removal of a refresh token cascade deletes the associated access + token in the db, which can make cache invalidation fail, if not handled + properly. This test will catch such behavior if it ever happens again. 1. User logs in with device1 2. User logs in with device2 and requests a refresh token - 3. Device2 sends a read marker (should work) + 3. Device2 calls /whoami (should work) 4. Device1 removes device2 - 5. Device2 tries to send a read marker (should fail) + 5. Device2 calls /whoami (should fail) """ # Login with device1 device1_tok = self.login("test", self.user_pass, device_id="device1") @@ -1563,7 +1561,7 @@ def test_token_invalid_after_refresh_token_issued_and_device_removed(self) -> No # Login with device2 and request a refresh token login_response = self.make_request( "POST", - "/_matrix/client/v3/login", + "/_matrix/client/r0/login", { "type": "m.login.password", "user": "test", @@ -1577,19 +1575,10 @@ def test_token_invalid_after_refresh_token_issued_and_device_removed(self) -> No device2_id = login_response.json_body["device_id"] self.assertEqual(device2_id, "device2") - # Create a room and send a message - room_id = self.helper.create_room_as(self.user, tok=device1_tok) - event_id = self.helper.send( - room_id=room_id, body="test message", tok=device1_tok - )["event_id"] - - # Device2 sends a read marker (should work) + # Device2 calls /whoami (should work) channel = self.make_request( - "POST", - f"/rooms/{room_id}/read_markers", - content={ - "m.fully_read": event_id, - }, + "GET", + "/_matrix/client/v3/account/whoami", access_token=device2_tok, ) self.assertEqual(channel.code, HTTPStatus.OK, channel.result) @@ -1622,17 +1611,14 @@ def test_token_invalid_after_refresh_token_issued_and_device_removed(self) -> No ) self.assertEqual(delete_channel.code, HTTPStatus.OK, delete_channel.result) - # Device2 tries to send a read marker (should fail) + # Device2 calls /whoami (should fail) channel = self.make_request( - "POST", - f"/rooms/{room_id}/read_markers", - content={ - "m.fully_read": event_id, - }, + "GET", + "/_matrix/client/v3/account/whoami", access_token=device2_tok, ) self.assertEqual(channel.code, HTTPStatus.UNAUTHORIZED, channel.result) - self.assertEqual(channel.json_body["errcode"], Codes.UNKNOWN_TOKEN) + self.assertEqual(channel.json_body["errcode"], "M_UNKNOWN_TOKEN") def oidc_config( diff --git a/tests/rest/client/test_capabilities.py b/tests/rest/client/test_capabilities.py index 2567fee2a4..c28e0605b5 100644 --- a/tests/rest/client/test_capabilities.py +++ b/tests/rest/client/test_capabilities.py @@ -28,7 +28,12 @@ from synapse.util.clock import Clock from tests import unittest -from tests.unittest import override_config +from tests.unittest import override_config, skip_unless + +try: + import lxml +except ImportError: + lxml = None # type: ignore[assignment] class CapabilitiesTestCase(unittest.HomeserverTestCase): @@ -276,3 +281,39 @@ def test_get_forget_forced_upon_leave_without_auto_forget(self) -> None: self.assertFalse( capabilities["org.matrix.msc4267.forget_forced_upon_leave"]["enabled"] ) + + @override_config( + { + "url_preview_enabled": False, + "experimental_features": {"msc4452_enabled": True}, + } + ) + def test_url_previews_disabled(self) -> None: + access_token = self.get_success( + self.auth_handler.create_access_token_for_user_id( + self.user, device_id=None, valid_until_ms=None + ) + ) + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertFalse(capabilities["io.element.msc4452.preview_url"]["enabled"]) + + @skip_unless(lxml is not None, "Requires lxml") + @override_config( + { + "url_preview_enabled": True, + "url_preview_ip_range_blacklist": ["127.0.0.1"], + "experimental_features": {"msc4452_enabled": True}, + }, + ) + def test_url_previews_enabled(self) -> None: + access_token = self.get_success( + self.auth_handler.create_access_token_for_user_id( + self.user, device_id=None, valid_until_ms=None + ) + ) + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertTrue(capabilities["io.element.msc4452.preview_url"]["enabled"]) diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index ec81b1413c..3409581c5e 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -1573,6 +1573,60 @@ def test_blocked_url(self) -> None: self.assertEqual(channel.code, 403, channel.result) +# We test this here because this endpoint must still work +# even if lxml is not installed. +class URLPreviewDisabledTests(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + media.register_servlets, + ] + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self.register_user("user", "password") + self.tok = self.login("user", "password") + + @override_config( + { + "url_preview_enabled": False, + "experimental_features": {"msc4452_enabled": False}, + } + ) + def test_disabled_previews(self) -> None: + """Tests that disabling URL previews gives back a sane response.""" + channel = self.make_request( + "GET", + "/_matrix/client/v1/media/preview_url?url=" + quote("http://example.com"), + access_token=self.tok, + ) + self.assertEqual(channel.code, 404, channel.result) + self.assertEqual( + channel.json_body, + {"errcode": "M_UNRECOGNIZED", "error": "Unrecognized request"}, + ) + + @override_config( + { + "url_preview_enabled": False, + "experimental_features": {"msc4452_enabled": True}, + } + ) + def test_disabled_previews_with_msc4452(self) -> None: + """Tests that disabling URL previews gives back a sane response.""" + channel = self.make_request( + "GET", + "/_matrix/client/v1/media/preview_url?url=" + quote("http://example.com"), + access_token=self.tok, + ) + self.assertEqual(channel.code, 403, channel.result) + self.assertEqual( + channel.json_body, + {"errcode": "M_FORBIDDEN", "error": "URL Previews are disabled"}, + ) + + class MediaConfigTest(unittest.HomeserverTestCase): servlets = [ media.register_servlets, diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 61e7e87f62..10325c536a 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -59,7 +59,7 @@ sync, ) from synapse.server import HomeServer -from synapse.types import JsonDict, RoomAlias, UserID, create_requester +from synapse.types import JsonDict, JsonMapping, RoomAlias, UserID, create_requester from synapse.util.clock import Clock from synapse.util.stringutils import random_string @@ -1859,7 +1859,7 @@ class SpamCheck: mock_return_value: str | bool | Codes | tuple[Codes, JsonDict] | bool = ( "NOT_SPAM" ) - mock_content: JsonDict | None = None + mock_content: JsonMapping | None = None async def check_event_for_spam( self, @@ -2248,7 +2248,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.room_id = self.helper.create_room_as(self.user_id) def test_topo_token_is_accepted(self) -> None: - token = "t1-0_0_0_0_0_0_0_0_0_0_0_0" + token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) @@ -2259,7 +2259,7 @@ def test_topo_token_is_accepted(self) -> None: self.assertTrue("end" in channel.json_body) def test_stream_token_is_accepted_for_fwd_pagianation(self) -> None: - token = "s0_0_0_0_0_0_0_0_0_0_0_0" + token = "s0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) diff --git a/tests/rest/client/test_third_party_rules.py b/tests/rest/client/test_third_party_rules.py index 9373303666..fdf138b4ef 100644 --- a/tests/rest/client/test_third_party_rules.py +++ b/tests/rest/client/test_third_party_rules.py @@ -628,6 +628,7 @@ def _send_event_over_federation(self) -> None: "origin_server_ts": self.clock.time_msec(), "prev_events": [], "auth_events": [], + "hashes": {}, "signatures": {}, "unsigned": {}, } diff --git a/tests/server.py b/tests/server.py index 1b6e8633fc..8cea391fb3 100644 --- a/tests/server.py +++ b/tests/server.py @@ -101,6 +101,7 @@ from synapse.storage.prepare_database import prepare_database from synapse.types import ISynapseReactor, JsonDict from synapse.util.clock import Clock +from synapse.util.json import json_encoder from tests.utils import ( LEAVE_DB, @@ -422,7 +423,7 @@ def make_request( path = b"/" + path if isinstance(content, dict): - content = json.dumps(content).encode("utf8") + content = json_encoder.encode(content).encode("utf8") if isinstance(content, str): content = content.encode("utf8") diff --git a/tests/state/test_v2.py b/tests/state/test_v2.py index 85ce5bede2..8b3b919f44 100644 --- a/tests/state/test_v2.py +++ b/tests/state/test_v2.py @@ -33,7 +33,7 @@ from synapse.api.constants import EventTypes, JoinRules, Membership from synapse.api.room_versions import RoomVersions from synapse.event_auth import auth_types_for_event -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.state.v2 import ( _get_auth_chain_difference, _get_power_level_for_sender, @@ -45,6 +45,7 @@ from synapse.util.duration import Duration from tests import unittest +from tests.test_utils.event_builders import make_test_event ALICE = "@alice:example.com" BOB = "@bob:example.com" @@ -119,7 +120,7 @@ def to_event(self, auth_events: list[str], prev_events: list[str]) -> EventBase: if self.state_key is not None: event_dict["state_key"] = self.state_key - return make_event_from_dict(event_dict) + return make_test_event(event_dict) # All graphs start with this set of events @@ -878,7 +879,7 @@ def test_get_power_level_for_sender(self) -> None: on room version""" store = TestStateResolutionStore({}) for room_version in [RoomVersions.V10, RoomVersions.V11]: - create_event = make_event_from_dict( + create_event = make_test_event( { "room_id": ROOM_ID, "sender": ALICE, @@ -894,9 +895,9 @@ def test_get_power_level_for_sender(self) -> None: else {} ), }, - room_version, + room_version=room_version, ) - member_event = make_event_from_dict( + member_event = make_test_event( { "room_id": ROOM_ID, "sender": ALICE, @@ -908,9 +909,9 @@ def test_get_power_level_for_sender(self) -> None: "auth_events": [create_event.event_id], "prev_events": [create_event.event_id], }, - room_version, + room_version=room_version, ) - pl_event = make_event_from_dict( + pl_event = make_test_event( { "room_id": ROOM_ID, "sender": ALICE, @@ -926,7 +927,7 @@ def test_get_power_level_for_sender(self) -> None: "auth_events": [create_event.event_id, member_event.event_id], "prev_events": [member_event.event_id], }, - room_version, + room_version=room_version, ) event_map = { @@ -940,7 +941,7 @@ def test_get_power_level_for_sender(self) -> None: CHARLIE: 10, } for user_id, want_pl in want_pls.items(): - test_event = make_event_from_dict( + test_event = make_test_event( { "room_id": ROOM_ID, "sender": user_id, @@ -954,7 +955,7 @@ def test_get_power_level_for_sender(self) -> None: ], "prev_events": [pl_event.event_id], }, - room_version, + room_version=room_version, ) event_map[test_event.event_id] = test_event got_pl = self.successResultOf( @@ -977,7 +978,7 @@ def test_get_power_level_for_sender(self) -> None: CHARLIE: 0, } for user_id, want_pl in want_pls.items(): - test_event = make_event_from_dict( + test_event = make_test_event( { "room_id": ROOM_ID, "sender": user_id, @@ -991,7 +992,7 @@ def test_get_power_level_for_sender(self) -> None: ], "prev_events": [pl_event.event_id], }, - room_version, + room_version=room_version, ) got_pl = self.successResultOf( defer.ensureDeferred( diff --git a/tests/state/test_v21.py b/tests/state/test_v21.py index 9f607176c9..6b4d2837ab 100644 --- a/tests/state/test_v21.py +++ b/tests/state/test_v21.py @@ -495,6 +495,7 @@ def create_event( "prev_events": prev_events, "auth_events": auth_events, "origin_server_ts": monotonic_timestamp(), + "hashes": {}, } if event_type != EventTypes.Create: if room_id is None: diff --git a/tests/storage/databases/main/test_events_worker.py b/tests/storage/databases/main/test_events_worker.py index c786271c09..dd647d29d6 100644 --- a/tests/storage/databases/main/test_events_worker.py +++ b/tests/storage/databases/main/test_events_worker.py @@ -28,7 +28,6 @@ from twisted.internet.testing import MemoryReactor from synapse.api.room_versions import EventFormatVersions, RoomVersions -from synapse.events import make_event_from_dict from synapse.logging.context import LoggingContext from synapse.rest import admin from synapse.rest.client import login, room @@ -42,6 +41,7 @@ from synapse.util.clock import Clock from tests import unittest +from tests.test_utils.event_builders import make_test_event from tests.test_utils.event_injection import create_event, inject_event @@ -377,7 +377,7 @@ def _populate_events(self) -> None: "type": f"test {idx}", "room_id": self.room_id, } - event = make_event_from_dict(event_json, room_version=RoomVersions.V4) + event = make_test_event(event_json, room_version=RoomVersions.V4) event_id = event.event_id self.get_success( self.store.db_pool.simple_upsert( @@ -400,7 +400,7 @@ def _populate_events(self) -> None: {"event_id": event_id}, { "room_id": self.room_id, - "json": json.dumps(event_json), + "json": json.dumps(event.get_dict()), "internal_metadata": "{}", "format_version": EventFormatVersions.ROOM_V4_PLUS, }, diff --git a/tests/storage/databases/main/test_lock.py b/tests/storage/databases/main/test_lock.py index 622eb96ded..c38e0cc834 100644 --- a/tests/storage/databases/main/test_lock.py +++ b/tests/storage/databases/main/test_lock.py @@ -26,7 +26,7 @@ from twisted.internet.testing import MemoryReactor from synapse.server import HomeServer -from synapse.storage.databases.main.lock import _LOCK_TIMEOUT_MS, _RENEWAL_INTERVAL +from synapse.storage.databases.main.lock import _LOCK_TIMEOUT, _RENEWAL_INTERVAL from synapse.util.clock import Clock from tests import unittest @@ -117,7 +117,7 @@ def test_maintain_lock(self) -> None: self.get_success(lock.__aenter__()) # Wait for ages with the lock, we should not be able to get the lock. - self.reactor.advance(5 * _LOCK_TIMEOUT_MS / 1000) + self.reactor.advance(5 * _LOCK_TIMEOUT.as_secs()) lock2 = self.get_success(self.store.try_acquire_lock("name", "key")) self.assertIsNone(lock2) @@ -138,7 +138,7 @@ def test_timeout_lock(self) -> None: lock._looping_call.stop() # Wait for the lock to timeout. - self.reactor.advance(2 * _LOCK_TIMEOUT_MS / 1000) + self.reactor.advance(2 * _LOCK_TIMEOUT.as_secs()) lock2 = self.get_success(self.store.try_acquire_lock("name", "key")) self.assertIsNotNone(lock2) @@ -154,7 +154,7 @@ def test_drop(self) -> None: del lock # Wait for the lock to timeout. - self.reactor.advance(2 * _LOCK_TIMEOUT_MS / 1000) + self.reactor.advance(2 * _LOCK_TIMEOUT.as_secs()) lock2 = self.get_success(self.store.try_acquire_lock("name", "key")) self.assertIsNotNone(lock2) @@ -402,7 +402,7 @@ def test_timeout_lock(self) -> None: lock._looping_call.stop() # Wait for the lock to timeout. - self.reactor.advance(2 * _LOCK_TIMEOUT_MS / 1000) + self.reactor.advance(2 * _LOCK_TIMEOUT.as_secs()) lock2 = self.get_success( self.store.try_acquire_read_write_lock("name", "key", write=True) @@ -422,7 +422,7 @@ def test_drop(self) -> None: del lock # Wait for the lock to timeout. - self.reactor.advance(2 * _LOCK_TIMEOUT_MS / 1000) + self.reactor.advance(2 * _LOCK_TIMEOUT.as_secs()) lock2 = self.get_success( self.store.try_acquire_read_write_lock("name", "key", write=True) diff --git a/tests/storage/test_events.py b/tests/storage/test_events.py index 7d1c96f97f..c511ac9252 100644 --- a/tests/storage/test_events.py +++ b/tests/storage/test_events.py @@ -26,13 +26,13 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.room_versions import RoomVersions from synapse.events import EventBase -from synapse.federation.federation_base import event_from_pdu_json from synapse.rest import admin from synapse.rest.client import login, room from synapse.server import HomeServer from synapse.types import StateMap from synapse.util.clock import Clock +from tests.test_utils.event_builders import make_test_pdu_event from tests.unittest import HomeserverTestCase logger = logging.getLogger(__name__) @@ -142,7 +142,7 @@ def prepare( # Fudge a remote event and persist it. This will be the extremity before # the gap. - self.remote_event_1 = event_from_pdu_json( + self.remote_event_1 = make_test_pdu_event( { "type": EventTypes.Message, "state_key": "@user:other", @@ -198,7 +198,7 @@ def test_prune_gap(self) -> None: # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other", @@ -237,7 +237,7 @@ def test_do_not_prune_gap_if_state_different(self) -> None: ) # Fudge a second event which points to an event we don't have. - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Message, "state_key": "@user:other", @@ -286,7 +286,7 @@ def test_prune_gap_if_old(self) -> None: # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other2", @@ -322,7 +322,7 @@ def test_do_not_prune_gap_if_other_server(self) -> None: # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other2", @@ -367,7 +367,7 @@ def test_prune_gap_if_dummy_remote(self) -> None: # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other2", @@ -415,7 +415,7 @@ def test_prune_gap_if_dummy_local(self) -> None: # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other2", @@ -455,7 +455,7 @@ def test_do_not_prune_gap_if_not_dummy(self) -> None: # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other2", @@ -514,7 +514,7 @@ def test_remote_user_rooms_cache_invalidated(self) -> None: # Fudge a join event for a remote user. remote_user = "@user:other" - remote_event_1 = event_from_pdu_json( + remote_event_1 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": remote_user, @@ -561,7 +561,7 @@ def test_room_remote_user_cache_invalidated(self) -> None: # Fudge a join event for a remote user. remote_user = "@user:other" - remote_event_1 = event_from_pdu_json( + remote_event_1 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": remote_user, diff --git a/tests/storage/test_msc4242_state_dag.py b/tests/storage/test_msc4242_state_dag.py index 8775e5c8eb..2150bc0996 100644 --- a/tests/storage/test_msc4242_state_dag.py +++ b/tests/storage/test_msc4242_state_dag.py @@ -19,12 +19,13 @@ from synapse.api.constants import EventTypes from synapse.api.errors import SynapseError from synapse.api.room_versions import RoomVersions -from synapse.events import FrozenEventVMSC4242, make_event_from_dict +from synapse.events.py_protocol import MSC4242Event, supports_msc4242_state_dag from synapse.events.snapshot import EventContext from synapse.rest.client import room from synapse.server import HomeServer from synapse.util.clock import Clock +from tests.test_utils.event_builders import make_test_event from tests.unittest import HomeserverTestCase, override_config @@ -152,8 +153,8 @@ def _make_event( id: str, prev_state_events: list[str], rejected: bool = False, - ) -> tuple[FrozenEventVMSC4242, EventContext]: - ev = make_event_from_dict( + ) -> tuple[MSC4242Event, EventContext]: + ev = make_test_event( { "prev_state_events": prev_state_events, "content": { @@ -166,8 +167,8 @@ def _make_event( }, room_version=RoomVersions.MSC4242v12, ) - assert isinstance(ev, FrozenEventVMSC4242) - ev._event_id = id + ev._event_id = id # type: ignore[attr-defined] + assert supports_msc4242_state_dag(ev) ctx = Mock() ctx.rejected = rejected return ev, ctx @@ -175,7 +176,7 @@ def _make_event( def _test( self, current_fwds: list[str], - new_events: list[tuple[FrozenEventVMSC4242, EventContext]], + new_events: list[tuple[MSC4242Event, EventContext]], want_new_extrems: set[str], want_raises: bool = False, ) -> None: diff --git a/tests/storage/test_sliding_sync_tables.py b/tests/storage/test_sliding_sync_tables.py index 39226ff9be..67b8d1138f 100644 --- a/tests/storage/test_sliding_sync_tables.py +++ b/tests/storage/test_sliding_sync_tables.py @@ -27,7 +27,7 @@ from synapse.api.constants import EventContentFields, EventTypes, Membership, RoomTypes from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, StrippedStateEvent, make_event_from_dict +from synapse.events import EventBase, StrippedStateEvent from synapse.events.snapshot import EventContext from synapse.rest import admin from synapse.rest.client import login, room, sync @@ -46,6 +46,7 @@ from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase +from tests.test_utils.event_builders import make_test_event from tests.test_utils.event_injection import create_event logger = logging.getLogger(__name__) @@ -240,7 +241,7 @@ def _retract_remote_invite_for_user( "prev_events": [], } - kick_event = make_event_from_dict( + kick_event = make_test_event( kick_event_dict, room_version=RoomVersions.V10, ) diff --git a/tests/synapse_rust/test_json_object.py b/tests/synapse_rust/test_json_object.py new file mode 100644 index 0000000000..77b188eee0 --- /dev/null +++ b/tests/synapse_rust/test_json_object.py @@ -0,0 +1,149 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from synapse.synapse_rust.events import JsonObject +from synapse.util import MutableOverlayMapping + +from tests import unittest + + +class JsonObjectMappingTestCase(unittest.TestCase): + def test_new_and_basic_mapping_behavior(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + self.assertEqual(len(obj), 2) + self.assertTrue("a" in obj) + self.assertTrue("b" in obj) + self.assertFalse("c" in obj) + self.assertFalse(123 in obj) # type: ignore[comparison-overlap] + + def test_getitem_and_key_errors(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + self.assertEqual(obj["a"], 1) + + with self.assertRaises(KeyError): + _ = obj["missing"] + + with self.assertRaises(KeyError): + _ = obj[10] # type: ignore[index] + + def test_iter_keys_values_items(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + iterator = iter(obj) + first = next(iterator) + second = next(iterator) + self.assertCountEqual((first, second), ("a", "b")) + with self.assertRaises(StopIteration): + next(iterator) + + self.assertCountEqual(list(obj.keys()), ["a", "b"]) + self.assertCountEqual(list(obj.values()), [1, 2]) + self.assertCountEqual(list(obj.items()), [("a", 1), ("b", 2)]) + + def test_keys_set_like_behavior(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + # Test 'and' operator. + self.assertEqual(obj.keys() & {"a"}, {"a"}) + self.assertEqual({"a"} & obj.keys(), {"a"}) + self.assertEqual(obj.keys() & {"c"}, set()) + self.assertEqual({"c"} & obj.keys(), set()) + + # Test 'or' operator. + self.assertEqual(obj.keys() | {"a"}, {"a", "b"}) + self.assertEqual({"a"} | obj.keys(), {"a", "b"}) + self.assertEqual(obj.keys() | {"c"}, {"a", "b", "c"}) + self.assertEqual({"c"} | obj.keys(), {"a", "b", "c"}) + + # Test 'xor' operator. + self.assertEqual(obj.keys() ^ {"a"}, {"b"}) + self.assertEqual({"a"} ^ obj.keys(), {"b"}) + self.assertEqual(obj.keys() ^ {"c"}, {"a", "b", "c"}) + self.assertEqual({"c"} ^ obj.keys(), {"a", "b", "c"}) + + # Test 'sub' operator. + self.assertEqual(obj.keys() - {"a"}, {"b"}) + self.assertEqual({"a"} - obj.keys(), set()) + self.assertEqual(obj.keys() - {"c"}, {"a", "b"}) + self.assertEqual({"c"} - obj.keys(), {"c"}) + + def test_values_view(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + values = obj.values() + + self.assertEqual(len(values), 2) + self.assertCountEqual(list(values), [1, 2]) + + self.assertIn(1, values) + self.assertIn(2, values) + self.assertNotIn(3, values) + self.assertNotIn("a", values) + self.assertNotIn(object(), values) + + # Iterating twice should yield the same values. + self.assertCountEqual(list(values), [1, 2]) + + def test_items_view(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + items = obj.items() + + self.assertEqual(len(items), 2) + self.assertCountEqual(list(items), [("a", 1), ("b", 2)]) + + self.assertIn(("a", 1), items) + self.assertIn(("b", 2), items) + self.assertNotIn(("a", 2), items) + self.assertNotIn(("c", 1), items) + self.assertNotIn("a", items) + self.assertNotIn(("a", 1, "extra"), items) + + # Iterating twice should yield the same items. + self.assertCountEqual(list(items), [("a", 1), ("b", 2)]) + + def test_get(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + self.assertEqual(obj.get("a"), 1) + self.assertEqual(obj.get("missing", "fallback"), "fallback") + self.assertEqual(obj.get(5, "fallback"), "fallback") # type: ignore[call-overload] + + def test_eq(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + self.assertEqual(obj, {"a": 1, "b": 2}) + self.assertNotEqual(obj, {"a": 1}) + self.assertNotEqual(obj, ["a", "b"]) + + def test_str_and_repr(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + self.assertEqual(str(obj), r'{"a":1,"b":2}') + self.assertEqual(repr(obj), r'JsonObject({"a":1,"b":2})') + + def test_json_object_constructor(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + # Passing in an existing JsonObject should work. + obj2 = JsonObject(obj) + self.assertEqual(obj2, {"a": 1, "b": 2}) + + # Other mapping types should also work. + obj3 = JsonObject(MutableOverlayMapping({"a": 1, "b": 2})) + self.assertEqual(obj3, {"a": 1, "b": 2}) + + # Test that passing a non-mapping raises a TypeError. + with self.assertRaises(TypeError): + JsonObject(123) # type: ignore[arg-type] diff --git a/tests/test_event_auth.py b/tests/test_event_auth.py index 9258f0d4dc..4537186ee6 100644 --- a/tests/test_event_auth.py +++ b/tests/test_event_auth.py @@ -29,11 +29,12 @@ from synapse.api.constants import EventContentFields, RejectedReason from synapse.api.errors import AuthError, SynapseError from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions -from synapse.events import EventBase, event_exists_in_state_dag, make_event_from_dict +from synapse.events import EventBase, event_exists_in_state_dag from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import JsonDict, get_domain_from_id from tests.test_utils import get_awaitable_result +from tests.test_utils.event_builders import make_test_event class _StubEventSourceStore: @@ -128,7 +129,7 @@ def test_create_event_with_prev_events(self) -> None: # we make both a good event and a bad event, to check that we are rejecting # the bad event for the reason we think we are. - good_event = make_event_from_dict( + good_event = make_test_event( { "room_id": TEST_ROOM_ID, "type": "m.room.create", @@ -143,7 +144,7 @@ def test_create_event_with_prev_events(self) -> None: }, room_version=RoomVersions.V9, ) - bad_event = make_event_from_dict( + bad_event = make_test_event( {**good_event.get_dict(), "prev_events": ["$fakeevent"]}, room_version=RoomVersions.V9, ) @@ -387,7 +388,7 @@ def test_msc4242_state_dag_rules(self) -> None: creator = "@creator:example.com" room_version = RoomVersions.MSC4242v12 - create_event = make_event_from_dict( + create_event = make_test_event( { "type": "m.room.create", "sender": creator, @@ -398,7 +399,7 @@ def test_msc4242_state_dag_rules(self) -> None: }, room_version, ) - create_event_2 = make_event_from_dict( + create_event_2 = make_test_event( { "type": "m.room.create", "sender": creator, @@ -411,7 +412,7 @@ def test_msc4242_state_dag_rules(self) -> None: ) room_id = create_event.room_id another_room_id = create_event_2.room_id - join_event = make_event_from_dict( + join_event = make_test_event( { "room_id": room_id, "type": "m.room.member", @@ -424,7 +425,7 @@ def test_msc4242_state_dag_rules(self) -> None: room_version, {"calculated_auth_event_ids": [create_event.event_id]}, ) - event_in_another_room = make_event_from_dict( + event_in_another_room = make_test_event( { "room_id": another_room_id, "type": "m.room.join_rules", @@ -437,7 +438,7 @@ def test_msc4242_state_dag_rules(self) -> None: room_version, {"calculated_auth_event_ids": [create_event.event_id, join_event.event_id]}, ) - msg_event = make_event_from_dict( + msg_event = make_test_event( { "room_id": room_id, "type": "m.room.message", @@ -449,7 +450,7 @@ def test_msc4242_state_dag_rules(self) -> None: room_version, {"calculated_auth_event_ids": [create_event.event_id, join_event.event_id]}, ) - rejected_event = make_event_from_dict( + rejected_event = make_test_event( { "room_id": room_id, "type": "m.room.name", @@ -470,7 +471,7 @@ def test_msc4242_state_dag_rules(self) -> None: RejectingTestCase( name="create event has prev_state_events", events_in_store=[], - test_event=make_event_from_dict( + test_event=make_test_event( { "type": "m.room.create", "sender": creator, @@ -486,7 +487,7 @@ def test_msc4242_state_dag_rules(self) -> None: RejectingTestCase( name="prev_state_event belongs in a different room", events_in_store=[create_event, join_event, event_in_another_room], - test_event=make_event_from_dict( + test_event=make_test_event( { "room_id": room_id, "type": "m.room.name", @@ -508,7 +509,7 @@ def test_msc4242_state_dag_rules(self) -> None: RejectingTestCase( name="prev_state_event is a message event", events_in_store=[create_event, join_event, msg_event], - test_event=make_event_from_dict( + test_event=make_test_event( { "room_id": room_id, "type": "m.room.name", @@ -530,7 +531,7 @@ def test_msc4242_state_dag_rules(self) -> None: RejectingTestCase( name="prev_state_event was rejected", events_in_store=[create_event, join_event, rejected_event], - test_event=make_event_from_dict( + test_event=make_test_event( { "room_id": room_id, "type": "m.room.name", @@ -892,7 +893,7 @@ def test_join_rules_msc3083_restricted(self) -> None: def test_room_v10_rejects_string_power_levels(self) -> None: pl_event_content = {"users_default": "42"} - pl_event = make_event_from_dict( + pl_event = make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(RoomVersions.V10), @@ -906,7 +907,7 @@ def test_room_v10_rejects_string_power_levels(self) -> None: ) pl_event2_content = {"events": {"m.room.name": "42", "m.room.power_levels": 42}} - pl_event2 = make_event_from_dict( + pl_event2 = make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(RoomVersions.V10), @@ -936,7 +937,7 @@ def test_room_v10_rejects_other_non_integer_power_levels(self) -> None: """ def create_event(pl_event_content: dict[str, Any]) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(RoomVersions.V10), @@ -1049,7 +1050,7 @@ def check_events(events: list[dict], should_exist: bool) -> None: "signatures": {"test.com": {"ed25519:0": "some9signature"}}, } base.update(ev) - event = make_event_from_dict(base, RoomVersions.V10) + event = make_test_event(base, RoomVersions.V10) got = event_exists_in_state_dag(event) self.assertEqual( got, should_exist, f"{ev} should_exist={should_exist} but got {got}" @@ -1068,7 +1069,7 @@ def _create_event( room_version: RoomVersion, user_id: str, ) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(room_version), @@ -1089,7 +1090,7 @@ def _member_event( sender: str | None = None, additional_content: dict | None = None, ) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(room_version), @@ -1122,7 +1123,7 @@ def _power_levels_event( sender: str, content: JsonDict, ) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(room_version), @@ -1145,7 +1146,7 @@ def _alias_event(room_version: RoomVersion, sender: str, **kwargs: Any) -> Event "content": {"aliases": []}, } data.update(**kwargs) - return make_event_from_dict(data, room_version=room_version) + return make_test_event(data, room_version=room_version) def _build_auth_dict_for_room_version( @@ -1164,7 +1165,7 @@ def _random_state_event( ) -> EventBase: if auth_events is None: auth_events = [] - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(room_version), @@ -1181,7 +1182,7 @@ def _random_state_event( def _join_rules_event( room_version: RoomVersion, sender: str, join_rule: str ) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(room_version), diff --git a/tests/test_notifier.py b/tests/test_notifier.py new file mode 100644 index 0000000000..c65134e832 --- /dev/null +++ b/tests/test_notifier.py @@ -0,0 +1,136 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +import logging + +from twisted.internet import defer +from twisted.internet.testing import MemoryReactor + +from synapse.server import HomeServer +from synapse.types import MultiWriterStreamToken, StreamKeyType, StreamToken +from synapse.util.clock import Clock +from synapse.util.duration import Duration + +import tests.unittest + +logger = logging.getLogger(__name__) + + +class NotifierTestCase(tests.unittest.HomeserverTestCase): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = self.hs.get_datastores().main + self.notifier = self.hs.get_notifier() + + def test_wait_for_stream_token_with_caught_up_token(self) -> None: + """ + Test `wait_for_stream_token` when we receive a token that we are caught up to. + """ + # Create a token + receipt_id_gen = self.store.get_receipts_stream_id_gen() + receipt_token = MultiWriterStreamToken.from_generator(receipt_id_gen) + token = StreamToken.START.copy_and_replace(StreamKeyType.RECEIPT, receipt_token) + + # Function under test + wait_d = defer.ensureDeferred(self.notifier.wait_for_stream_token(token)) + + # Done waiting and caught-up (True) + wait_result = self.get_success(wait_d) + self.assertEqual(wait_result, True) + + def test_wait_for_stream_token_with_future_sync_token(self) -> None: + """ + Test `wait_for_stream_token` when we receive a token that is ahead of our + current token, we'll wait until the stream position advances. + + This can happen if replication streams start lagging, and the client's + previous sync request was serviced by a worker ahead of ours. + """ + # We simulate a lagging stream by getting a stream ID from the ID gen + # and then waiting to mark it as "persisted". + receipt_id_gen = self.store.get_receipts_stream_id_gen() + ctx_mgr = receipt_id_gen.get_next() + receipt_stream_id = self.get_success(ctx_mgr.__aenter__()) + + # Create the new token based on the stream ID above. + current_receipt_token = MultiWriterStreamToken.from_generator(receipt_id_gen) + receipt_token = current_receipt_token.copy_and_advance( + MultiWriterStreamToken(stream=receipt_stream_id) + ) + token = StreamToken.START.copy_and_advance(StreamKeyType.RECEIPT, receipt_token) + + # Function under test + wait_d = defer.ensureDeferred(self.notifier.wait_for_stream_token(token)) + + # This should block waiting for the stream to update + # + # Advance time a little bit to make the + # `wait_for_stream_token(...)` sleep loop iterate. + self.reactor.advance(Duration(seconds=2).as_secs()) + # It should still not be done yet + self.assertFalse(wait_d.called) + + # Marking the stream ID as persisted should unblock the request. + self.get_success(ctx_mgr.__aexit__(None, None, None)) + + # Advance time to make another iteration of + # `wait_for_stream_token(...)` sleep loop so it sees that we're + # finally caught up now. + self.reactor.advance(Duration(seconds=1).as_secs()) + + # Done waiting and caught-up (True) + wait_result = self.get_success(wait_d) + self.assertEqual(wait_result, True) + + def test_wait_for_stream_token_with_future_sync_token_timeout( + self, + ) -> None: + """ + Test `wait_for_stream_token` when we receive a token that is ahead of our + current token, we'll wait until the stream position advances *until* we hit the + timeout. + + This can happen if replication streams start lagging, and the client's + previous sync request was serviced by a worker ahead of ours. + """ + # We simulate a lagging stream by getting a stream ID from the ID gen + # and then waiting to mark it as "persisted". + receipt_id_gen = self.store.get_receipts_stream_id_gen() + ctx_mgr = receipt_id_gen.get_next() + receipt_stream_id = self.get_success(ctx_mgr.__aenter__()) + + # Create the new token based on the stream ID above. + current_receipt_token = MultiWriterStreamToken.from_generator(receipt_id_gen) + receipt_token = current_receipt_token.copy_and_advance( + MultiWriterStreamToken(stream=receipt_stream_id) + ) + token = StreamToken.START.copy_and_advance(StreamKeyType.RECEIPT, receipt_token) + + # Function under test + wait_d = defer.ensureDeferred(self.notifier.wait_for_stream_token(token)) + # Advance time a little bit to make the + # `wait_for_stream_token(...)` sleep loop record 0 as the `start` time. + self.reactor.advance(Duration(seconds=0).as_secs()) + + # This should block waiting for the stream to update + # + # Advance time a little bit to make the + # `wait_for_stream_token(...)` sleep loop iterate. + self.reactor.advance(Duration(seconds=5).as_secs()) + # It should still not be done yet (not enough time to hit the timeout) + self.assertFalse(wait_d.called) + # Advance time past the 10 second timeout (5 + 6 = 11 seconds) to make the + # `wait_for_stream_token(...)` sleep loop give up. + self.reactor.advance(Duration(seconds=6).as_secs()) + + # Make sure we gave up waiting and not caught-up (False) + wait_result = self.get_success(wait_d) + self.assertEqual(wait_result, False) diff --git a/tests/test_state.py b/tests/test_state.py index 7df95ebf8b..0a6720d6ef 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -32,15 +32,16 @@ from synapse.api.auth.internal import InternalAuth from synapse.api.constants import EventTypes, Membership from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.events.snapshot import EventContext from synapse.state import StateHandler, StateResolutionHandler, _make_state_cache_entry -from synapse.types import MutableStateMap, StateMap +from synapse.types import JsonDict, MutableStateMap, StateMap from synapse.types.state import StateFilter from synapse.util.macaroons import MacaroonGenerator from tests import unittest from tests.server import get_clock +from tests.test_utils.event_builders import make_test_event from tests.utils import default_config _next_event_id = 1000 @@ -67,13 +68,14 @@ def create_event( else: name = "<%s, %s>" % (type, event_id) - d = { + d: JsonDict = { "event_id": event_id, "type": type, "sender": "@user_id:example.com", "room_id": "!room_id:example.com", "depth": depth, "prev_events": prev_events or [], + "content": {}, } if state_key is not None: @@ -81,7 +83,7 @@ def create_event( d.update(kwargs) - return make_event_from_dict(d) + return make_test_event(d) class _DummyStore: diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py index 0df5a4e6c3..4170768208 100644 --- a/tests/test_utils/__init__.py +++ b/tests/test_utils/__init__.py @@ -24,7 +24,6 @@ """ import base64 -import json import sys import warnings from binascii import unhexlify @@ -41,6 +40,7 @@ from twisted.web.iweb import IResponse from synapse.types import JsonSerializable +from synapse.util.json import json_encoder if TYPE_CHECKING: from sys import UnraisableHookArgs @@ -127,7 +127,7 @@ def deliverBody(self, protocol: IProtocol) -> None: @classmethod def json(cls, *, code: int = 200, payload: JsonSerializable) -> "FakeResponse": headers = Headers({"Content-Type": ["application/json"]}) - body = json.dumps(payload).encode("utf-8") + body = json_encoder.encode(payload).encode("utf-8") return cls(code=code, body=body, headers=headers) diff --git a/tests/test_utils/event_builders.py b/tests/test_utils/event_builders.py new file mode 100644 index 0000000000..a8eb586c1f --- /dev/null +++ b/tests/test_utils/event_builders.py @@ -0,0 +1,117 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from typing import TypedDict + +from typing_extensions import NotRequired, Unpack + +from synapse.api.room_versions import ( + RoomVersion, + RoomVersions, +) +from synapse.events import EventBase, make_event_from_dict +from synapse.federation.federation_base import event_from_pdu_json +from synapse.types import JsonDict + + +def default_event_fields(room_version: RoomVersion) -> JsonDict: + """Return default values for every field required by `room_version`.""" + + # We need to include entries for every required field for the room version. + # Note that they don't necessarily have to be valid values, just enough to + # allow us to construct the event class. (Ideally we'd build a fully valid + # event, but this is fine for now.) + defaults: JsonDict = { + "type": "m.test", + "sender": "@test:test", + "content": {}, + "depth": 1, + "origin_server_ts": 1, + "hashes": {}, + "prev_events": [], + "room_id": "!test:test", + } + + # MSC4242 versions require prev_state_events but not auth_events. + if room_version.msc4242_state_dags: + defaults["prev_state_events"] = [] + else: + defaults["auth_events"] = [] + + if room_version == RoomVersions.V1: + # V1 requires an event_id field, but later versions don't. + defaults["event_id"] = "$test_event_id:matrix.org" + + return defaults + + +def make_test_event( + event_dict: JsonDict | None = None, + room_version: RoomVersion = RoomVersions.V1, + internal_metadata_dict: JsonDict | None = None, + rejected_reason: str | None = None, + **fields: Unpack["_EventFields"], +) -> EventBase: + """Build an `EventBase` with defaults for the strict-required fields. + + Pass an `event_dict` and/or `**fields` keyword arguments — both are + merged on top of the format-version defaults from + `default_event_fields`. Explicit values win over defaults, and + `**fields` wins over `event_dict` so call sites can override a + shared base dict with one-off tweaks. + """ + merged: JsonDict = { + **default_event_fields(room_version), + **(event_dict or {}), + **fields, + } + return make_event_from_dict( + merged, + room_version=room_version, + internal_metadata_dict=internal_metadata_dict, + rejected_reason=rejected_reason, + ) + + +def make_test_pdu_event( + pdu: JsonDict, + room_version: RoomVersion, + received_time: int | None = None, +) -> EventBase: + """Wrapper around `event_from_pdu_json` for test PDU dicts. + + Federation-side test fixtures often omit fields the strict Rust ctor + requires (e.g. `hashes`, `auth_events`, `prev_events`, `depth`) + because those tests focus on transport/auth flow rather than event + well-formedness. This helper layers in the same format-version + defaults as `make_test_event` before delegating. + """ + pdu = {**default_event_fields(room_version), **pdu} + return event_from_pdu_json(pdu, room_version, received_time=received_time) + + +class _EventFields(TypedDict): + """Type for `kwargs` in `make_test_event`.""" + + event_id: NotRequired[str] + type: NotRequired[str] + sender: NotRequired[str] + content: NotRequired[JsonDict] + depth: NotRequired[int] + origin_server_ts: NotRequired[int] + hashes: NotRequired[dict[str, str]] + auth_events: NotRequired[list[str]] + prev_events: NotRequired[list[str]] + prev_state_events: NotRequired[list[str]] + room_id: NotRequired[str]