diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
index c3638c35eb..839b895c82 100644
--- a/.git-blame-ignore-revs
+++ b/.git-blame-ignore-revs
@@ -21,4 +21,8 @@ aff1eb7c671b0a3813407321d2702ec46c71fa56
0a00b7ff14890987f09112a2ae696c61001e6cf1
# Convert tests/rest/admin/test_room.py to unix file endings (#7953).
-c4268e3da64f1abb5b31deaeb5769adb6510c0a7
\ No newline at end of file
+c4268e3da64f1abb5b31deaeb5769adb6510c0a7
+
+# Update black to 23.1.0 (#15103)
+9bb2eac71962970d02842bca441f4bcdbbf93a11
+
diff --git a/.github/workflows/docs-pr-netlify.yaml b/.github/workflows/docs-pr-netlify.yaml
index ef7a38144e..1704b3ce93 100644
--- a/.github/workflows/docs-pr-netlify.yaml
+++ b/.github/workflows/docs-pr-netlify.yaml
@@ -14,7 +14,7 @@ jobs:
# There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
# (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
- name: 📥 Download artifact
- uses: dawidd6/action-download-artifact@bd10f381a96414ce2b13a11bfa89902ba7cea07f # v2.24.3
+ uses: dawidd6/action-download-artifact@b59d8c6a6c5c6c6437954f470d963c0b20ea7415 # v2.25.0
with:
workflow: docs-pr.yaml
run_id: ${{ github.event.workflow_run.id }}
diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml
index 8485daf87f..6da7c22e4c 100644
--- a/.github/workflows/latest_deps.yml
+++ b/.github/workflows/latest_deps.yml
@@ -27,7 +27,7 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Install Rust
- uses: dtolnay/rust-toolchain@25dc93b901a87e864900a8aec6c12e9aa794c0c3
+ uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
with:
toolchain: stable
- uses: Swatinem/rust-cache@v2
@@ -61,7 +61,7 @@ jobs:
- uses: actions/checkout@v3
- name: Install Rust
- uses: dtolnay/rust-toolchain@25dc93b901a87e864900a8aec6c12e9aa794c0c3
+ uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
with:
toolchain: stable
- uses: Swatinem/rust-cache@v2
@@ -134,7 +134,7 @@ jobs:
- uses: actions/checkout@v3
- name: Install Rust
- uses: dtolnay/rust-toolchain@25dc93b901a87e864900a8aec6c12e9aa794c0c3
+ uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
with:
toolchain: stable
- uses: Swatinem/rust-cache@v2
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 94f7f2657c..cfafeaadc9 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -112,7 +112,7 @@ jobs:
# There don't seem to be versioned releases of this action per se: for each rust
# version there is a branch which gets constantly rebased on top of master.
# We pin to a specific commit for paranoia's sake.
- uses: dtolnay/rust-toolchain@25dc93b901a87e864900a8aec6c12e9aa794c0c3
+ uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
with:
toolchain: 1.58.1
components: clippy
@@ -134,7 +134,7 @@ jobs:
# There don't seem to be versioned releases of this action per se: for each rust
# version there is a branch which gets constantly rebased on top of master.
# We pin to a specific commit for paranoia's sake.
- uses: dtolnay/rust-toolchain@25dc93b901a87e864900a8aec6c12e9aa794c0c3
+ uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
with:
toolchain: nightly-2022-12-01
components: clippy
@@ -154,7 +154,7 @@ jobs:
# There don't seem to be versioned releases of this action per se: for each rust
# version there is a branch which gets constantly rebased on top of master.
# We pin to a specific commit for paranoia's sake.
- uses: dtolnay/rust-toolchain@25dc93b901a87e864900a8aec6c12e9aa794c0c3
+ uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
with:
toolchain: 1.58.1
components: rustfmt
@@ -221,7 +221,7 @@ jobs:
# There don't seem to be versioned releases of this action per se: for each rust
# version there is a branch which gets constantly rebased on top of master.
# We pin to a specific commit for paranoia's sake.
- uses: dtolnay/rust-toolchain@25dc93b901a87e864900a8aec6c12e9aa794c0c3
+ uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
with:
toolchain: 1.58.1
- uses: Swatinem/rust-cache@v2
@@ -266,7 +266,7 @@ jobs:
# There don't seem to be versioned releases of this action per se: for each rust
# version there is a branch which gets constantly rebased on top of master.
# We pin to a specific commit for paranoia's sake.
- uses: dtolnay/rust-toolchain@25dc93b901a87e864900a8aec6c12e9aa794c0c3
+ uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
with:
toolchain: 1.58.1
- uses: Swatinem/rust-cache@v2
@@ -386,7 +386,7 @@ jobs:
# There don't seem to be versioned releases of this action per se: for each rust
# version there is a branch which gets constantly rebased on top of master.
# We pin to a specific commit for paranoia's sake.
- uses: dtolnay/rust-toolchain@25dc93b901a87e864900a8aec6c12e9aa794c0c3
+ uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
with:
toolchain: 1.58.1
- uses: Swatinem/rust-cache@v2
@@ -531,7 +531,7 @@ jobs:
# There don't seem to be versioned releases of this action per se: for each rust
# version there is a branch which gets constantly rebased on top of master.
# We pin to a specific commit for paranoia's sake.
- uses: dtolnay/rust-toolchain@25dc93b901a87e864900a8aec6c12e9aa794c0c3
+ uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
with:
toolchain: 1.58.1
- uses: Swatinem/rust-cache@v2
@@ -562,7 +562,7 @@ jobs:
# There don't seem to be versioned releases of this action per se: for each rust
# version there is a branch which gets constantly rebased on top of master.
# We pin to a specific commit for paranoia's sake.
- uses: dtolnay/rust-toolchain@25dc93b901a87e864900a8aec6c12e9aa794c0c3
+ uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
with:
toolchain: 1.58.1
- uses: Swatinem/rust-cache@v2
@@ -585,7 +585,7 @@ jobs:
# There don't seem to be versioned releases of this action per se: for each rust
# version there is a branch which gets constantly rebased on top of master.
# We pin to a specific commit for paranoia's sake.
- uses: dtolnay/rust-toolchain@25dc93b901a87e864900a8aec6c12e9aa794c0c3
+ uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
with:
toolchain: nightly-2022-12-01
- uses: Swatinem/rust-cache@v2
diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml
index 5654d2f3e2..db514571c4 100644
--- a/.github/workflows/twisted_trunk.yml
+++ b/.github/workflows/twisted_trunk.yml
@@ -18,7 +18,7 @@ jobs:
- uses: actions/checkout@v3
- name: Install Rust
- uses: dtolnay/rust-toolchain@25dc93b901a87e864900a8aec6c12e9aa794c0c3
+ uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
with:
toolchain: stable
- uses: Swatinem/rust-cache@v2
@@ -43,7 +43,7 @@ jobs:
- run: sudo apt-get -qq install xmlsec1
- name: Install Rust
- uses: dtolnay/rust-toolchain@25dc93b901a87e864900a8aec6c12e9aa794c0c3
+ uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
with:
toolchain: stable
- uses: Swatinem/rust-cache@v2
@@ -82,7 +82,7 @@ jobs:
- uses: actions/checkout@v3
- name: Install Rust
- uses: dtolnay/rust-toolchain@25dc93b901a87e864900a8aec6c12e9aa794c0c3
+ uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
with:
toolchain: stable
- uses: Swatinem/rust-cache@v2
diff --git a/CHANGES.md b/CHANGES.md
index a62bd4eb28..f5c19bcb97 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,91 @@
+Synapse 1.78.0rc1 (2023-02-21)
+==============================
+
+Features
+--------
+
+- Implement the experimental `exact_event_match` push rule condition from [MSC3758](https://github.com/matrix-org/matrix-spec-proposals/pull/3758). ([\#14964](https://github.com/matrix-org/synapse/issues/14964))
+- Add account data to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.78/usage/administration/admin_faq.html#how-can-i-export-user-data). ([\#14969](https://github.com/matrix-org/synapse/issues/14969))
+- Implement [MSC3873](https://github.com/matrix-org/matrix-spec-proposals/pull/3873) to disambiguate push rule keys with dots in them. ([\#15004](https://github.com/matrix-org/synapse/issues/15004))
+- Allow Synapse to use a specific Redis [logical database](https://redis.io/commands/select/) in worker-mode deployments. ([\#15034](https://github.com/matrix-org/synapse/issues/15034))
+- Tag opentracing spans for federation requests with the name of the worker serving the request. ([\#15042](https://github.com/matrix-org/synapse/issues/15042))
+- Implement the experimental `exact_event_property_contains` push rule condition from [MSC3966](https://github.com/matrix-org/matrix-spec-proposals/pull/3966). ([\#15045](https://github.com/matrix-org/synapse/issues/15045))
+- Remove spurious `dont_notify` action from the defaults for the `.m.rule.reaction` pushrule. ([\#15073](https://github.com/matrix-org/synapse/issues/15073))
+- Update the error code returned when user sends a duplicate annotation. ([\#15075](https://github.com/matrix-org/synapse/issues/15075))
+
+
+Bugfixes
+--------
+
+- Prevent clients from reporting nonexistent events. ([\#13779](https://github.com/matrix-org/synapse/issues/13779))
+- Return spec-compliant JSON errors when unknown endpoints are requested. ([\#14605](https://github.com/matrix-org/synapse/issues/14605))
+- Fix a long-standing bug where the room aliases returned could be corrupted. ([\#15038](https://github.com/matrix-org/synapse/issues/15038))
+- Fix a bug introduced in Synapse 1.76.0 where partially-joined rooms could not be deleted using the [purge room API](https://matrix-org.github.io/synapse/latest/admin_api/rooms.html#delete-room-api). ([\#15068](https://github.com/matrix-org/synapse/issues/15068))
+- Fix a long-standing bug where federated joins would fail if the first server in the list of servers to try is not in the room. ([\#15074](https://github.com/matrix-org/synapse/issues/15074))
+- Fix a bug introduced in Synapse v1.74.0 where searching with colons when using ICU for search term tokenisation would fail with an error. ([\#15079](https://github.com/matrix-org/synapse/issues/15079))
+- Reduce the likelihood of a rare race condition where rejoining a restricted room over federation would fail. ([\#15080](https://github.com/matrix-org/synapse/issues/15080))
+- Fix a bug introduced in Synapse 1.76 where workers would fail to start if the `health` listener was configured. ([\#15096](https://github.com/matrix-org/synapse/issues/15096))
+- Fix a bug introduced in Synapse 1.75 where the [portdb script](https://matrix-org.github.io/synapse/release-v1.78/postgres.html#porting-from-sqlite) would fail to run after a room had been faster-joined. ([\#15108](https://github.com/matrix-org/synapse/issues/15108))
+
+
+Improved Documentation
+----------------------
+
+- Document how to start Synapse with Poetry. Contributed by @thezaidbintariq. ([\#14892](https://github.com/matrix-org/synapse/issues/14892), [\#15022](https://github.com/matrix-org/synapse/issues/15022))
+- Update delegation documentation to clarify that SRV DNS delegation does not eliminate all needs to serve files from .well-known locations. Contributed by @williamkray. ([\#14959](https://github.com/matrix-org/synapse/issues/14959))
+- Fix a mistake in registration_shared_secret_path docs. ([\#15078](https://github.com/matrix-org/synapse/issues/15078))
+- Refer to a more recent blog post on the [Database Maintenance Tools](https://matrix-org.github.io/synapse/latest/usage/administration/database_maintenance_tools.html) page. Contributed by @jahway603. ([\#15083](https://github.com/matrix-org/synapse/issues/15083))
+
+
+Internal Changes
+----------------
+
+- Re-type hint some collections as read-only. ([\#13755](https://github.com/matrix-org/synapse/issues/13755))
+- Faster joins: don't stall when another user joins during a partial-state room resync. ([\#14606](https://github.com/matrix-org/synapse/issues/14606))
+- Add a class `UnpersistedEventContext` to allow for the batching up of storing state groups. ([\#14675](https://github.com/matrix-org/synapse/issues/14675))
+- Add a check to ensure that locked dependencies have source distributions available. ([\#14742](https://github.com/matrix-org/synapse/issues/14742))
+- Tweak comment on `_is_local_room_accessible` as part of room visibility in `/hierarchy` to clarify the condition for a room being visible. ([\#14834](https://github.com/matrix-org/synapse/issues/14834))
+- Prevent `WARNING: there is already a transaction in progress` lines appearing in PostgreSQL's logs on some occasions. ([\#14840](https://github.com/matrix-org/synapse/issues/14840))
+- Use `StrCollection` to avoid potential bugs with `Collection[str]`. ([\#14929](https://github.com/matrix-org/synapse/issues/14929))
+- Improve performance of `/sync` in a few situations. ([\#14973](https://github.com/matrix-org/synapse/issues/14973))
+- Limit concurrent event creation for a room to avoid state resolution when sending bursts of events to a local room. ([\#14977](https://github.com/matrix-org/synapse/issues/14977))
+- Skip calculating unread push actions in /sync when enable_push is false. ([\#14980](https://github.com/matrix-org/synapse/issues/14980))
+- Add a schema dump symlinks inside `contrib`, to make it easier for IDEs to interrogate Synapse's database schema. ([\#14982](https://github.com/matrix-org/synapse/issues/14982))
+- Improve type hints. ([\#15008](https://github.com/matrix-org/synapse/issues/15008), [\#15026](https://github.com/matrix-org/synapse/issues/15026), [\#15027](https://github.com/matrix-org/synapse/issues/15027), [\#15028](https://github.com/matrix-org/synapse/issues/15028), [\#15031](https://github.com/matrix-org/synapse/issues/15031), [\#15035](https://github.com/matrix-org/synapse/issues/15035), [\#15052](https://github.com/matrix-org/synapse/issues/15052), [\#15072](https://github.com/matrix-org/synapse/issues/15072), [\#15084](https://github.com/matrix-org/synapse/issues/15084))
+- Update [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952) support based on changes to the MSC. ([\#15037](https://github.com/matrix-org/synapse/issues/15037))
+- Avoid mutating a cached value in `get_user_devices_from_cache`. ([\#15040](https://github.com/matrix-org/synapse/issues/15040))
+- Fix a rare exception in logs on start up. ([\#15041](https://github.com/matrix-org/synapse/issues/15041))
+- Update pyo3-log to v0.8.1. ([\#15043](https://github.com/matrix-org/synapse/issues/15043))
+- Avoid mutating cached values in `_generate_sync_entry_for_account_data`. ([\#15047](https://github.com/matrix-org/synapse/issues/15047))
+- Refactor arguments of `try_unbind_threepid` and `_try_unbind_threepid_with_id_server` to not use dictionaries. ([\#15053](https://github.com/matrix-org/synapse/issues/15053))
+- Merge debug logging from the hotfixes branch. ([\#15054](https://github.com/matrix-org/synapse/issues/15054))
+- Faster joins: omit device list updates originating from partial state rooms in /sync responses without lazy loading of members enabled. ([\#15069](https://github.com/matrix-org/synapse/issues/15069))
+- Fix clashing database transaction name. ([\#15070](https://github.com/matrix-org/synapse/issues/15070))
+- Upper-bound frozendict dependency. This works around us being unable to test installing our wheels against Python 3.11 in CI. ([\#15114](https://github.com/matrix-org/synapse/issues/15114))
+- Tweak logging for when a worker waits for its view of a replication stream to catch up. ([\#15120](https://github.com/matrix-org/synapse/issues/15120))
+
+<details><summary>Locked dependency updates</summary>
+
+- Bump bleach from 5.0.1 to 6.0.0. ([\#15059](https://github.com/matrix-org/synapse/issues/15059))
+- Bump cryptography from 38.0.4 to 39.0.1. ([\#15020](https://github.com/matrix-org/synapse/issues/15020))
+- Bump ruff version from 0.0.230 to 0.0.237. ([\#15033](https://github.com/matrix-org/synapse/issues/15033))
+- Bump dtolnay/rust-toolchain from 9cd00a88a73addc8617065438eff914dd08d0955 to 25dc93b901a87e864900a8aec6c12e9aa794c0c3. ([\#15060](https://github.com/matrix-org/synapse/issues/15060))
+- Bump systemd-python from 234 to 235. ([\#15061](https://github.com/matrix-org/synapse/issues/15061))
+- Bump serde_json from 1.0.92 to 1.0.93. ([\#15062](https://github.com/matrix-org/synapse/issues/15062))
+- Bump types-requests from 2.28.11.8 to 2.28.11.12. ([\#15063](https://github.com/matrix-org/synapse/issues/15063))
+- Bump types-pillow from 9.4.0.5 to 9.4.0.10. ([\#15064](https://github.com/matrix-org/synapse/issues/15064))
+- Bump sentry-sdk from 1.13.0 to 1.15.0. ([\#15065](https://github.com/matrix-org/synapse/issues/15065))
+- Bump types-jsonschema from 4.17.0.3 to 4.17.0.5. ([\#15099](https://github.com/matrix-org/synapse/issues/15099))
+- Bump types-bleach from 5.0.3.1 to 6.0.0.0. ([\#15100](https://github.com/matrix-org/synapse/issues/15100))
+- Bump dtolnay/rust-toolchain from 25dc93b901a87e864900a8aec6c12e9aa794c0c3 to e12eda571dc9a5ee5d58eecf4738ec291c66f295. ([\#15101](https://github.com/matrix-org/synapse/issues/15101))
+- Bump dawidd6/action-download-artifact from 2.24.3 to 2.25.0. ([\#15102](https://github.com/matrix-org/synapse/issues/15102))
+- Bump types-pillow from 9.4.0.10 to 9.4.0.13. ([\#15104](https://github.com/matrix-org/synapse/issues/15104))
+- Bump types-setuptools from 67.1.0.0 to 67.3.0.1. ([\#15105](https://github.com/matrix-org/synapse/issues/15105))
+
+
+</details>
+
+
Synapse 1.77.0 (2023-02-14)
===========================
@@ -63,7 +151,7 @@ Internal Changes
- Preparatory work for adding a denormalised event stream ordering column in the future. Contributed by Nick @ Beeper (@fizzadar). ([\#14979](https://github.com/matrix-org/synapse/issues/14979), [9cd7610](https://github.com/matrix-org/synapse/commit/9cd7610f86ab5051c9365dd38d1eec405a5f8ca6), [f10caa7](https://github.com/matrix-org/synapse/commit/f10caa73eee0caa91cf373966104d1ededae2aee); see [\#15014](https://github.com/matrix-org/synapse/issues/15014))
- Add tests for `_flatten_dict`. ([\#14981](https://github.com/matrix-org/synapse/issues/14981), [\#15002](https://github.com/matrix-org/synapse/issues/15002))
-<details><summary>Dependabot updates</summary>
+<details><summary>Locked dependency updates</summary>
- Bump dtolnay/rust-toolchain from e645b0cf01249a964ec099494d38d2da0f0b349f to 9cd00a88a73addc8617065438eff914dd08d0955. ([\#14968](https://github.com/matrix-org/synapse/issues/14968))
- Bump docker/build-push-action from 3 to 4. ([\#14952](https://github.com/matrix-org/synapse/issues/14952))
diff --git a/changelog.d/13755.misc b/changelog.d/13755.misc
deleted file mode 100644
index 662ee00e99..0000000000
--- a/changelog.d/13755.misc
+++ /dev/null
@@ -1 +0,0 @@
-Re-type hint some collections as read-only.
diff --git a/changelog.d/13779.bugfix b/changelog.d/13779.bugfix
deleted file mode 100644
index a92c722c6e..0000000000
--- a/changelog.d/13779.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Prevent clients from reporting nonexistent events.
\ No newline at end of file
diff --git a/changelog.d/14605.bugfix b/changelog.d/14605.bugfix
deleted file mode 100644
index cb95a87d92..0000000000
--- a/changelog.d/14605.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Return spec-compliant JSON errors when unknown endpoints are requested.
diff --git a/changelog.d/14606.misc b/changelog.d/14606.misc
deleted file mode 100644
index e2debc96d8..0000000000
--- a/changelog.d/14606.misc
+++ /dev/null
@@ -1 +0,0 @@
-Faster joins: don't stall when another user joins during a fast join resync.
diff --git a/changelog.d/14675.misc b/changelog.d/14675.misc
deleted file mode 100644
index bc1ac1c82a..0000000000
--- a/changelog.d/14675.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add a class UnpersistedEventContext to allow for the batching up of storing state groups.
diff --git a/changelog.d/14742.misc b/changelog.d/14742.misc
deleted file mode 100644
index c0b5d2c062..0000000000
--- a/changelog.d/14742.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add check to ensure locked dependencies have source distributions available.
\ No newline at end of file
diff --git a/changelog.d/14834.misc b/changelog.d/14834.misc
deleted file mode 100644
index e683212dc4..0000000000
--- a/changelog.d/14834.misc
+++ /dev/null
@@ -1 +0,0 @@
-Tweak comment on `_is_local_room_accessible` as part of room visibility in `/hierarchy` to clarify the condition for a room being visible.
\ No newline at end of file
diff --git a/changelog.d/14840.misc b/changelog.d/14840.misc
deleted file mode 100644
index ff6084284a..0000000000
--- a/changelog.d/14840.misc
+++ /dev/null
@@ -1 +0,0 @@
-Prevent "WARNING: there is already a transaction in progress" lines appearing in PostgreSQL's logs on some occasions.
\ No newline at end of file
diff --git a/changelog.d/14892.doc b/changelog.d/14892.doc
deleted file mode 100644
index 2bc3ad06c6..0000000000
--- a/changelog.d/14892.doc
+++ /dev/null
@@ -1 +0,0 @@
-Document how to start Synapse with Poetry. Contributed by @thezaidbintariq.
diff --git a/changelog.d/14929.misc b/changelog.d/14929.misc
deleted file mode 100644
index 2cc3614dfd..0000000000
--- a/changelog.d/14929.misc
+++ /dev/null
@@ -1 +0,0 @@
-Use `StrCollection` to avoid potential bugs with `Collection[str]`.
diff --git a/changelog.d/14959.doc b/changelog.d/14959.doc
deleted file mode 100644
index 45edf1a765..0000000000
--- a/changelog.d/14959.doc
+++ /dev/null
@@ -1 +0,0 @@
-Update delegation documentation to clarify that SRV DNS delegation does not eliminate all needs to serve files from .well-known locations. Contributed by @williamkray.
diff --git a/changelog.d/14964.feature b/changelog.d/14964.feature
deleted file mode 100644
index 13c0bc193b..0000000000
--- a/changelog.d/14964.feature
+++ /dev/null
@@ -1 +0,0 @@
-Implement the experimental `exact_event_match` push rule condition from [MSC3758](https://github.com/matrix-org/matrix-spec-proposals/pull/3758).
diff --git a/changelog.d/14969.feature b/changelog.d/14969.feature
deleted file mode 100644
index a4680ef9c8..0000000000
--- a/changelog.d/14969.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add account data to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.78/usage/administration/admin_faq.html#how-can-i-export-user-data).
\ No newline at end of file
diff --git a/changelog.d/14973.misc b/changelog.d/14973.misc
deleted file mode 100644
index 3657623602..0000000000
--- a/changelog.d/14973.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve performance of `/sync` in a few situations.
diff --git a/changelog.d/14977.misc b/changelog.d/14977.misc
deleted file mode 100644
index 4d551c52b7..0000000000
--- a/changelog.d/14977.misc
+++ /dev/null
@@ -1 +0,0 @@
-Limit concurrent event creation for a room to avoid state resolution when sending bursts of events to a local room.
\ No newline at end of file
diff --git a/changelog.d/14980.misc b/changelog.d/14980.misc
deleted file mode 100644
index 145f4a788b..0000000000
--- a/changelog.d/14980.misc
+++ /dev/null
@@ -1 +0,0 @@
-Skip calculating unread push actions in /sync when enable_push is false.
diff --git a/changelog.d/14982.misc b/changelog.d/14982.misc
deleted file mode 100644
index 9aaa7ce264..0000000000
--- a/changelog.d/14982.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add a schema dump symlinks inside `contrib`, to make it easier for IDEs to interrogate Synapse's database schema.
diff --git a/changelog.d/15004.feature b/changelog.d/15004.feature
deleted file mode 100644
index d11d0aca91..0000000000
--- a/changelog.d/15004.feature
+++ /dev/null
@@ -1 +0,0 @@
-Implement [MSC3873](https://github.com/matrix-org/matrix-spec-proposals/pull/3873) to unambiguate push rule keys with dots in them.
diff --git a/changelog.d/15008.misc b/changelog.d/15008.misc
deleted file mode 100644
index 93ceaeafc9..0000000000
--- a/changelog.d/15008.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve type hints.
diff --git a/changelog.d/15020.misc b/changelog.d/15020.misc
deleted file mode 100644
index c5290283f0..0000000000
--- a/changelog.d/15020.misc
+++ /dev/null
@@ -1 +0,0 @@
-Bump cryptography from 38.0.4 to 39.0.1.
diff --git a/changelog.d/15022.doc b/changelog.d/15022.doc
deleted file mode 100644
index e1627c20cb..0000000000
--- a/changelog.d/15022.doc
+++ /dev/null
@@ -1 +0,0 @@
-Document how to start Synapse in the contributing guide.
diff --git a/changelog.d/15026.misc b/changelog.d/15026.misc
deleted file mode 100644
index 93ceaeafc9..0000000000
--- a/changelog.d/15026.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve type hints.
diff --git a/changelog.d/15027.misc b/changelog.d/15027.misc
deleted file mode 100644
index 93ceaeafc9..0000000000
--- a/changelog.d/15027.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve type hints.
diff --git a/changelog.d/15028.misc b/changelog.d/15028.misc
deleted file mode 100644
index 93ceaeafc9..0000000000
--- a/changelog.d/15028.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve type hints.
diff --git a/changelog.d/15031.misc b/changelog.d/15031.misc
deleted file mode 100644
index 93ceaeafc9..0000000000
--- a/changelog.d/15031.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve type hints.
diff --git a/changelog.d/15033.misc b/changelog.d/15033.misc
deleted file mode 100644
index 83dc3a75b6..0000000000
--- a/changelog.d/15033.misc
+++ /dev/null
@@ -1 +0,0 @@
-Bump ruff version from 0.0.230 to 0.0.237.
diff --git a/changelog.d/15034.feature b/changelog.d/15034.feature
deleted file mode 100644
index 34f320da92..0000000000
--- a/changelog.d/15034.feature
+++ /dev/null
@@ -1 +0,0 @@
-Allow Synapse to use a specific Redis [logical database](https://redis.io/commands/select/) in worker-mode deployments.
diff --git a/changelog.d/15035.misc b/changelog.d/15035.misc
deleted file mode 100644
index 93ceaeafc9..0000000000
--- a/changelog.d/15035.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve type hints.
diff --git a/changelog.d/15037.misc b/changelog.d/15037.misc
deleted file mode 100644
index fabfe77d35..0000000000
--- a/changelog.d/15037.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952) support based on changes to the MSC.
diff --git a/changelog.d/15038.bugfix b/changelog.d/15038.bugfix
deleted file mode 100644
index 4695a09756..0000000000
--- a/changelog.d/15038.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a long-standing bug where the room aliases returned could be corrupted.
diff --git a/changelog.d/15040.misc b/changelog.d/15040.misc
deleted file mode 100644
index ca129b64af..0000000000
--- a/changelog.d/15040.misc
+++ /dev/null
@@ -1 +0,0 @@
-Avoid mutating a cached value in `get_user_devices_from_cache`.
diff --git a/changelog.d/15041.misc b/changelog.d/15041.misc
deleted file mode 100644
index d602b0043a..0000000000
--- a/changelog.d/15041.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix a rare exception in logs on start up.
diff --git a/changelog.d/15042.feature b/changelog.d/15042.feature
deleted file mode 100644
index 7a4de89f00..0000000000
--- a/changelog.d/15042.feature
+++ /dev/null
@@ -1 +0,0 @@
-Tag opentracing spans for federation requests with the name of the worker serving the request.
diff --git a/changelog.d/15043.misc b/changelog.d/15043.misc
deleted file mode 100644
index cb18394123..0000000000
--- a/changelog.d/15043.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update pyo3-log to v0.8.1.
diff --git a/changelog.d/15045.feature b/changelog.d/15045.feature
deleted file mode 100644
index 87766befda..0000000000
--- a/changelog.d/15045.feature
+++ /dev/null
@@ -1 +0,0 @@
-Experimental support for [MSC3966](https://github.com/matrix-org/matrix-spec-proposals/pull/3966): the `exact_event_property_contains` push rule condition.
diff --git a/changelog.d/15047.misc b/changelog.d/15047.misc
deleted file mode 100644
index 561dc874de..0000000000
--- a/changelog.d/15047.misc
+++ /dev/null
@@ -1 +0,0 @@
-Avoid mutating cached values in `_generate_sync_entry_for_account_data`.
diff --git a/changelog.d/15052.misc b/changelog.d/15052.misc
deleted file mode 100644
index 93ceaeafc9..0000000000
--- a/changelog.d/15052.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve type hints.
diff --git a/changelog.d/15053.misc b/changelog.d/15053.misc
deleted file mode 100644
index c27528f5c6..0000000000
--- a/changelog.d/15053.misc
+++ /dev/null
@@ -1 +0,0 @@
-Refactor arguments of `try_unbind_threepid` and `_try_unbind_threepid_with_id_server` to not use dictionaries.
\ No newline at end of file
diff --git a/changelog.d/15054.misc b/changelog.d/15054.misc
deleted file mode 100644
index d800b107cf..0000000000
--- a/changelog.d/15054.misc
+++ /dev/null
@@ -1 +0,0 @@
-Merge debug logging from the hotfixes branch.
diff --git a/changelog.d/15059.misc b/changelog.d/15059.misc
deleted file mode 100644
index e962b208fd..0000000000
--- a/changelog.d/15059.misc
+++ /dev/null
@@ -1 +0,0 @@
-Bump bleach from 5.0.1 to 6.0.0.
diff --git a/changelog.d/15060.misc b/changelog.d/15060.misc
deleted file mode 100644
index 5b99e06003..0000000000
--- a/changelog.d/15060.misc
+++ /dev/null
@@ -1 +0,0 @@
-Bump dtolnay/rust-toolchain from 9cd00a88a73addc8617065438eff914dd08d0955 to 25dc93b901a87e864900a8aec6c12e9aa794c0c3.
diff --git a/changelog.d/15061.misc b/changelog.d/15061.misc
deleted file mode 100644
index 40017827a2..0000000000
--- a/changelog.d/15061.misc
+++ /dev/null
@@ -1 +0,0 @@
-Bump systemd-python from 234 to 235.
diff --git a/changelog.d/15062.misc b/changelog.d/15062.misc
deleted file mode 100644
index adc1940630..0000000000
--- a/changelog.d/15062.misc
+++ /dev/null
@@ -1 +0,0 @@
-Bump serde_json from 1.0.92 to 1.0.93.
diff --git a/changelog.d/15063.misc b/changelog.d/15063.misc
deleted file mode 100644
index b52e1faed0..0000000000
--- a/changelog.d/15063.misc
+++ /dev/null
@@ -1 +0,0 @@
-Bump types-requests from 2.28.11.8 to 2.28.11.12.
diff --git a/changelog.d/15064.misc b/changelog.d/15064.misc
deleted file mode 100644
index 644d4bb230..0000000000
--- a/changelog.d/15064.misc
+++ /dev/null
@@ -1 +0,0 @@
-Bump types-pillow from 9.4.0.5 to 9.4.0.10.
diff --git a/changelog.d/15065.misc b/changelog.d/15065.misc
deleted file mode 100644
index df2f9a773e..0000000000
--- a/changelog.d/15065.misc
+++ /dev/null
@@ -1 +0,0 @@
-Bump sentry-sdk from 1.13.0 to 1.15.0.
diff --git a/changelog.d/15068.bugfix b/changelog.d/15068.bugfix
deleted file mode 100644
index f09ffa2877..0000000000
--- a/changelog.d/15068.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.76.0 where partially-joined rooms could not be deleted using the [purge room API](https://matrix-org.github.io/synapse/latest/admin_api/rooms.html#delete-room-api).
diff --git a/changelog.d/15069.misc b/changelog.d/15069.misc
deleted file mode 100644
index e7a619ad2b..0000000000
--- a/changelog.d/15069.misc
+++ /dev/null
@@ -1 +0,0 @@
-Faster joins: omit device list updates originating from partial state rooms in /sync responses without lazy loading of members enabled.
diff --git a/changelog.d/15070.misc b/changelog.d/15070.misc
deleted file mode 100644
index 0f3244de9f..0000000000
--- a/changelog.d/15070.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix clashing database transaction name.
diff --git a/changelog.d/15071.doc b/changelog.d/15071.doc
new file mode 100644
index 0000000000..7fbaba3e8c
--- /dev/null
+++ b/changelog.d/15071.doc
@@ -0,0 +1 @@
+Clarify which worker processes the ThirdPartyRules' [`on_new_event`](https://matrix-org.github.io/synapse/v1.78/modules/third_party_rules_callbacks.html#on_new_event) module API callback runs on.
\ No newline at end of file
diff --git a/changelog.d/15072.misc b/changelog.d/15072.misc
deleted file mode 100644
index 93ceaeafc9..0000000000
--- a/changelog.d/15072.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve type hints.
diff --git a/changelog.d/15073.feature b/changelog.d/15073.feature
deleted file mode 100644
index 2889e3444f..0000000000
--- a/changelog.d/15073.feature
+++ /dev/null
@@ -1 +0,0 @@
-Remove spurious `dont_notify` action from the defaults for the `.m.rule.reaction` pushrule.
diff --git a/changelog.d/15074.bugfix b/changelog.d/15074.bugfix
deleted file mode 100644
index d1ceb4f4c8..0000000000
--- a/changelog.d/15074.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a long-standing bug where federated joins would fail if the first server in the list of servers to try is not in the room.
diff --git a/changelog.d/15075.feature b/changelog.d/15075.feature
deleted file mode 100644
index d25a7567a4..0000000000
--- a/changelog.d/15075.feature
+++ /dev/null
@@ -1,2 +0,0 @@
-Update the error code returned when user sends a duplicate annotation.
-
diff --git a/changelog.d/15078.doc b/changelog.d/15078.doc
deleted file mode 100644
index 641f9a993b..0000000000
--- a/changelog.d/15078.doc
+++ /dev/null
@@ -1 +0,0 @@
-Fix a mistake in registration_shared_secret_path docs.
\ No newline at end of file
diff --git a/changelog.d/15080.bugfix b/changelog.d/15080.bugfix
deleted file mode 100644
index 965d0b921e..0000000000
--- a/changelog.d/15080.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Reduce the likelihood of a rare race condition where rejoining a restricted room over federation would fail.
diff --git a/changelog.d/15084.misc b/changelog.d/15084.misc
deleted file mode 100644
index 93ceaeafc9..0000000000
--- a/changelog.d/15084.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve type hints.
diff --git a/changelog.d/15092.bugfix b/changelog.d/15092.bugfix
new file mode 100644
index 0000000000..67509c5c69
--- /dev/null
+++ b/changelog.d/15092.bugfix
@@ -0,0 +1 @@
+Fix a long-standing bug where a URL preview would break if the discovered oEmbed failed to download.
diff --git a/changelog.d/15093.bugfix b/changelog.d/15093.bugfix
new file mode 100644
index 0000000000..00f1c19391
--- /dev/null
+++ b/changelog.d/15093.bugfix
@@ -0,0 +1 @@
+Remove the unspecced `room_alias` field from the [`/createRoom`](https://spec.matrix.org/v1.6/client-server-api/#post_matrixclientv3createroom) response.
diff --git a/changelog.d/15095.misc b/changelog.d/15095.misc
new file mode 100644
index 0000000000..a2fafe2fff
--- /dev/null
+++ b/changelog.d/15095.misc
@@ -0,0 +1 @@
+Refactor writing json data in `FileExfiltrationWriter`.
\ No newline at end of file
diff --git a/changelog.d/15103.misc b/changelog.d/15103.misc
new file mode 100644
index 0000000000..65322498c9
--- /dev/null
+++ b/changelog.d/15103.misc
@@ -0,0 +1 @@
+Bump black from 22.12.0 to 23.1.0.
diff --git a/changelog.d/15107.feature b/changelog.d/15107.feature
new file mode 100644
index 0000000000..2bdb6a29fc
--- /dev/null
+++ b/changelog.d/15107.feature
@@ -0,0 +1 @@
+Add media information to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.79/usage/administration/admin_faq.html#how-can-i-export-user-data).
\ No newline at end of file
diff --git a/changelog.d/15112.doc b/changelog.d/15112.doc
new file mode 100644
index 0000000000..7dec43a50b
--- /dev/null
+++ b/changelog.d/15112.doc
@@ -0,0 +1 @@
+Document using [Shibboleth](https://www.shibboleth.net/) as an OpenID Provider.
diff --git a/changelog.d/15135.misc b/changelog.d/15135.misc
new file mode 100644
index 0000000000..25c4dbffe1
--- /dev/null
+++ b/changelog.d/15135.misc
@@ -0,0 +1 @@
+Tighten the login ratelimit defaults.
diff --git a/changelog.d/15137.removal b/changelog.d/15137.removal
new file mode 100644
index 0000000000..c533b0c9dd
--- /dev/null
+++ b/changelog.d/15137.removal
@@ -0,0 +1 @@
+Remove the undocumented and unspecced `type` parameter to the `/thumbnail` endpoint.
diff --git a/changelog.d/15138.misc b/changelog.d/15138.misc
new file mode 100644
index 0000000000..fb706b27f2
--- /dev/null
+++ b/changelog.d/15138.misc
@@ -0,0 +1 @@
+Fix a typo in an experimental config setting.
diff --git a/changelog.d/15139.doc b/changelog.d/15139.doc
new file mode 100644
index 0000000000..d8ab48b272
--- /dev/null
+++ b/changelog.d/15139.doc
@@ -0,0 +1 @@
+Correct reference to `federation_verify_certificates` in configuration documentation.
diff --git a/debian/changelog b/debian/changelog
index ea651438f1..f9e95ee5e2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+matrix-synapse-py3 (1.78.0~rc1) stable; urgency=medium
+
+ * Add `matrix-org-archive-keyring` package as recommended.
+ * New Synapse release 1.78.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org> Tue, 21 Feb 2023 14:29:19 +0000
+
matrix-synapse-py3 (1.77.0) stable; urgency=medium
* New Synapse release 1.77.0.
diff --git a/debian/control b/debian/control
index bc628cec08..2ff55db5de 100644
--- a/debian/control
+++ b/debian/control
@@ -37,6 +37,7 @@ Depends:
# so we put perl:Depends in Suggests rather than Depends.
Recommends:
${shlibs1:Recommends},
+ matrix-org-archive-keyring,
Suggests:
sqlite3,
${perl:Depends},
diff --git a/docs/modules/third_party_rules_callbacks.md b/docs/modules/third_party_rules_callbacks.md
index e1a5b6524f..888e43bd10 100644
--- a/docs/modules/third_party_rules_callbacks.md
+++ b/docs/modules/third_party_rules_callbacks.md
@@ -146,6 +146,9 @@ Note that this callback is called when the event has already been processed and
into the room, which means this callback cannot be used to deny persisting the event. To
deny an incoming event, see [`check_event_for_spam`](spam_checker_callbacks.md#check_event_for_spam) instead.
+For any given event, this callback will be called on every worker process, even if that worker will not end up
+acting on that event. This callback will not be called for events that are marked as rejected.
+
If multiple modules implement this callback, Synapse runs them all in order.
### `check_can_shutdown_room`
diff --git a/docs/openid.md b/docs/openid.md
index 6ee8c83ec0..73f1e06121 100644
--- a/docs/openid.md
+++ b/docs/openid.md
@@ -590,6 +590,47 @@ oidc_providers:
Note that the fields `client_id` and `client_secret` are taken from the CURL response above.
+### Shibboleth with OIDC Plugin
+
+[Shibboleth](https://www.shibboleth.net/) is an open Standard IdP solution widely used by Universities.
+
+1. Shibboleth needs the [OIDC Plugin](https://shibboleth.atlassian.net/wiki/spaces/IDPPLUGINS/pages/1376878976/OIDC+OP) installed and working correctly.
+2. Create a new config on the IdP Side, ensure that the `client_id` and `client_secret`
+ are randomly generated data.
+```json
+{
+ "client_id": "SOME-CLIENT-ID",
+ "client_secret": "SOME-SUPER-SECRET-SECRET",
+ "response_types": ["code"],
+ "grant_types": ["authorization_code"],
+ "scope": "openid profile email",
+ "redirect_uris": ["https://[synapse public baseurl]/_synapse/client/oidc/callback"]
+}
+```
+
+Synapse config:
+
+```yaml
+oidc_providers:
+ # Shibboleth IDP
+ #
+ - idp_id: shibboleth
+ idp_name: "Shibboleth Login"
+ discover: true
+ issuer: "https://YOUR-IDP-URL.TLD"
+ client_id: "YOUR_CLIENT_ID"
+ client_secret: "YOUR-CLIENT-SECRECT-FROM-YOUR-IDP"
+ scopes: ["openid", "profile", "email"]
+ allow_existing_users: true
+ user_profile_method: "userinfo_endpoint"
+ user_mapping_provider:
+ config:
+ subject_claim: "sub"
+ localpart_template: "{{ user.sub.split('@')[0] }}"
+ display_name_template: "{{ user.name }}"
+ email_template: "{{ user.email }}"
+```
+
### Twitch
1. Setup a developer account on [Twitch](https://dev.twitch.tv/)
diff --git a/docs/usage/administration/admin_faq.md b/docs/usage/administration/admin_faq.md
index 925e1d175e..28c3dd53a5 100644
--- a/docs/usage/administration/admin_faq.md
+++ b/docs/usage/administration/admin_faq.md
@@ -70,13 +70,55 @@ output-directory
│ ├───state
│ ├───invite_state
│ └───knock_state
-└───user_data
- ├───account_data
- │ ├───global
- │ └───<room_id>
- ├───connections
- ├───devices
- └───profile
+├───user_data
+│ ├───account_data
+│ │ ├───global
+│ │ └───<room_id>
+│ ├───connections
+│ ├───devices
+│ └───profile
+└───media_ids
+ └───<media_id>
+```
+
+The `media_ids` folder contains only the metadata of the media uploaded by the user.
+It does not contain the media itself.
+Furthermore, only the `media_ids` that Synapse manages itself are exported.
+If another media repository (e.g. [matrix-media-repo](https://github.com/turt2live/matrix-media-repo))
+is used, the data must be exported separately.
+
+With the `media_ids` the media files can be downloaded.
+Media that have been sent in encrypted rooms are only retrieved in encrypted form.
+The following script can help with download the media files:
+
+```bash
+#!/usr/bin/env bash
+
+# Parameters
+#
+# source_directory: Directory which contains the export with the media_ids.
+# target_directory: Directory into which all files are to be downloaded.
+# repository_url: Address of the media repository resp. media worker.
+# serverName: Name of the server (`server_name` from homeserver.yaml).
+#
+# Example:
+# ./download_media.sh /tmp/export_data/media_ids/ /tmp/export_data/media_files/ http://localhost:8008 matrix.example.com
+
+source_directory=$1
+target_directory=$2
+repository_url=$3
+serverName=$4
+
+mkdir -p $target_directory
+
+for file in $source_directory/*; do
+ filename=$(basename ${file})
+ url=$repository_url/_matrix/media/v3/download/$serverName/$filename
+ echo "Downloading $filename - $url"
+ if ! wget -o /dev/null -P $target_directory $url; then
+ echo "Could not download $filename"
+ fi
+done
```
Manually resetting passwords
@@ -87,7 +129,7 @@ can reset a user's password using the [admin API](../../admin_api/user_admin_api
I have a problem with my server. Can I just delete my database and start again?
---
-Deleting your database is unlikely to make anything better.
+Deleting your database is unlikely to make anything better.
It's easy to make the mistake of thinking that you can start again from a clean
slate by dropping your database, but things don't work like that in a federated
@@ -102,7 +144,7 @@ Come and seek help in https://matrix.to/#/#synapse:matrix.org.
There are two exceptions when it might be sensible to delete your database and start again:
* You have *never* joined any rooms which are federated with other servers. For
-instance, a local deployment which the outside world can't talk to.
+instance, a local deployment which the outside world can't talk to.
* You are changing the `server_name` in the homeserver configuration. In effect
this makes your server a completely new one from the point of view of the network,
so in this case it makes sense to start with a clean database.
@@ -115,7 +157,7 @@ Using the following curl command:
curl -H 'Authorization: Bearer <access-token>' -X DELETE https://matrix.org/_matrix/client/r0/directory/room/<room-alias>
```
`<access-token>` - can be obtained in riot by looking in the riot settings, down the bottom is:
-Access Token:\<click to reveal\>
+Access Token:\<click to reveal\>
`<room-alias>` - the room alias, eg. #my_room:matrix.org this possibly needs to be URL encoded also, for example %23my_room%3Amatrix.org
@@ -152,13 +194,13 @@ What are the biggest rooms on my server?
---
```sql
-SELECT s.canonical_alias, g.room_id, count(*) AS num_rows
-FROM
- state_groups_state AS g,
- room_stats_state AS s
-WHERE g.room_id = s.room_id
+SELECT s.canonical_alias, g.room_id, count(*) AS num_rows
+FROM
+ state_groups_state AS g,
+ room_stats_state AS s
+WHERE g.room_id = s.room_id
GROUP BY s.canonical_alias, g.room_id
-ORDER BY num_rows desc
+ORDER BY num_rows desc
LIMIT 10;
```
diff --git a/docs/usage/administration/database_maintenance_tools.md b/docs/usage/administration/database_maintenance_tools.md
index 92b805d413..e19380db07 100644
--- a/docs/usage/administration/database_maintenance_tools.md
+++ b/docs/usage/administration/database_maintenance_tools.md
@@ -1,4 +1,4 @@
-This blog post by Victor Berger explains how to use many of the tools listed on this page: https://levans.fr/shrink-synapse-database.html
+_This [blog post by Jackson Chen](https://jacksonchen666.com/posts/2022-12-03/14-33-00/) (Dec 2022) explains how to use many of the tools listed on this page. There is also an [earlier blog by Victor Berger](https://levans.fr/shrink-synapse-database.html) (June 2020), though this may be outdated in places._
# List of useful tools and scripts for maintenance Synapse database:
@@ -15,4 +15,4 @@ The purge history API allows server admins to purge historic events from their d
Tool for compressing (deduplicating) `state_groups_state` table.
## [SQL for analyzing Synapse PostgreSQL database stats](useful_sql_for_admins.md)
-Some easy SQL that reports useful stats about your Synapse database.
\ No newline at end of file
+Some easy SQL that reports useful stats about your Synapse database.
diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md
index 58c6955689..4139961810 100644
--- a/docs/usage/configuration/config_documentation.md
+++ b/docs/usage/configuration/config_documentation.md
@@ -1105,7 +1105,7 @@ This setting should only be used in very specific cases, such as
federation over Tor hidden services and similar. For private networks
of homeservers, you likely want to use a private CA instead.
-Only effective if `federation_verify_certicates` is `true`.
+Only effective if `federation_verify_certificates` is `true`.
Example configuration:
```yaml
@@ -1518,11 +1518,11 @@ rc_registration_token_validity:
This option specifies several limits for login:
* `address` ratelimits login requests based on the client's IP
- address. Defaults to `per_second: 0.17`, `burst_count: 3`.
+ address. Defaults to `per_second: 0.003`, `burst_count: 5`.
* `account` ratelimits login requests based on the account the
- client is attempting to log into. Defaults to `per_second: 0.17`,
- `burst_count: 3`.
+ client is attempting to log into. Defaults to `per_second: 0.03`,
+ `burst_count: 5`.
* `failed_attempts` ratelimits login requests based on the account the
client is attempting to log into, based on the amount of failed login
diff --git a/poetry.lock b/poetry.lock
index eb1e3d797b..8ffdab7a22 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -90,32 +90,46 @@ typecheck = ["mypy"]
[[package]]
name = "black"
-version = "22.12.0"
+version = "23.1.0"
description = "The uncompromising code formatter."
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
- {file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"},
- {file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"},
- {file = "black-22.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d30b212bffeb1e252b31dd269dfae69dd17e06d92b87ad26e23890f3efea366f"},
- {file = "black-22.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:7412e75863aa5c5411886804678b7d083c7c28421210180d67dfd8cf1221e1f4"},
- {file = "black-22.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c116eed0efb9ff870ded8b62fe9f28dd61ef6e9ddd28d83d7d264a38417dcee2"},
- {file = "black-22.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1f58cbe16dfe8c12b7434e50ff889fa479072096d79f0a7f25e4ab8e94cd8350"},
- {file = "black-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d86c9f3db9b1bf6761244bc0b3572a546f5fe37917a044e02f3166d5aafa7d"},
- {file = "black-22.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:82d9fe8fee3401e02e79767016b4907820a7dc28d70d137eb397b92ef3cc5bfc"},
- {file = "black-22.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101c69b23df9b44247bd88e1d7e90154336ac4992502d4197bdac35dd7ee3320"},
- {file = "black-22.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:559c7a1ba9a006226f09e4916060982fd27334ae1998e7a38b3f33a37f7a2148"},
- {file = "black-22.12.0-py3-none-any.whl", hash = "sha256:436cc9167dd28040ad90d3b404aec22cedf24a6e4d7de221bec2730ec0c97bcf"},
- {file = "black-22.12.0.tar.gz", hash = "sha256:229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f"},
+ {file = "black-23.1.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:b6a92a41ee34b883b359998f0c8e6eb8e99803aa8bf3123bf2b2e6fec505a221"},
+ {file = "black-23.1.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:57c18c5165c1dbe291d5306e53fb3988122890e57bd9b3dcb75f967f13411a26"},
+ {file = "black-23.1.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:9880d7d419bb7e709b37e28deb5e68a49227713b623c72b2b931028ea65f619b"},
+ {file = "black-23.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e6663f91b6feca5d06f2ccd49a10f254f9298cc1f7f49c46e498a0771b507104"},
+ {file = "black-23.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:9afd3f493666a0cd8f8df9a0200c6359ac53940cbde049dcb1a7eb6ee2dd7074"},
+ {file = "black-23.1.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:bfffba28dc52a58f04492181392ee380e95262af14ee01d4bc7bb1b1c6ca8d27"},
+ {file = "black-23.1.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:c1c476bc7b7d021321e7d93dc2cbd78ce103b84d5a4cf97ed535fbc0d6660648"},
+ {file = "black-23.1.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:382998821f58e5c8238d3166c492139573325287820963d2f7de4d518bd76958"},
+ {file = "black-23.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bf649fda611c8550ca9d7592b69f0637218c2369b7744694c5e4902873b2f3a"},
+ {file = "black-23.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:121ca7f10b4a01fd99951234abdbd97728e1240be89fde18480ffac16503d481"},
+ {file = "black-23.1.0-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:a8471939da5e824b891b25751955be52ee7f8a30a916d570a5ba8e0f2eb2ecad"},
+ {file = "black-23.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8178318cb74f98bc571eef19068f6ab5613b3e59d4f47771582f04e175570ed8"},
+ {file = "black-23.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:a436e7881d33acaf2536c46a454bb964a50eff59b21b51c6ccf5a40601fbef24"},
+ {file = "black-23.1.0-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:a59db0a2094d2259c554676403fa2fac3473ccf1354c1c63eccf7ae65aac8ab6"},
+ {file = "black-23.1.0-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:0052dba51dec07ed029ed61b18183942043e00008ec65d5028814afaab9a22fd"},
+ {file = "black-23.1.0-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:49f7b39e30f326a34b5c9a4213213a6b221d7ae9d58ec70df1c4a307cf2a1580"},
+ {file = "black-23.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:162e37d49e93bd6eb6f1afc3e17a3d23a823042530c37c3c42eeeaf026f38468"},
+ {file = "black-23.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b70eb40a78dfac24842458476135f9b99ab952dd3f2dab738c1881a9b38b753"},
+ {file = "black-23.1.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:a29650759a6a0944e7cca036674655c2f0f63806ddecc45ed40b7b8aa314b651"},
+ {file = "black-23.1.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:bb460c8561c8c1bec7824ecbc3ce085eb50005883a6203dcfb0122e95797ee06"},
+ {file = "black-23.1.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:c91dfc2c2a4e50df0026f88d2215e166616e0c80e86004d0003ece0488db2739"},
+ {file = "black-23.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a951cc83ab535d248c89f300eccbd625e80ab880fbcfb5ac8afb5f01a258ac9"},
+ {file = "black-23.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:0680d4380db3719ebcfb2613f34e86c8e6d15ffeabcf8ec59355c5e7b85bb555"},
+ {file = "black-23.1.0-py3-none-any.whl", hash = "sha256:7a0f701d314cfa0896b9001df70a530eb2472babb76086344e688829efd97d32"},
+ {file = "black-23.1.0.tar.gz", hash = "sha256:b0bd97bea8903f5a2ba7219257a44e3f1f9d00073d6cc1add68f0beec69692ac"},
]
[package.dependencies]
click = ">=8.0.0"
mypy-extensions = ">=0.4.3"
+packaging = ">=22.0"
pathspec = ">=0.9.0"
platformdirs = ">=2"
-tomli = {version = ">=1.1.0", markers = "python_full_version < \"3.11.0a7\""}
+tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""}
typed-ast = {version = ">=1.4.2", markers = "python_version < \"3.8\" and implementation_name == \"cpython\""}
typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""}
@@ -1708,7 +1722,7 @@ files = [
cffi = ">=1.4.1"
[package.extras]
-docs = ["sphinx (>=1.6.5)", "sphinx_rtd_theme"]
+docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"]
tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"]
[[package]]
@@ -2548,14 +2562,14 @@ files = [
[[package]]
name = "types-bleach"
-version = "5.0.3.1"
+version = "6.0.0.0"
description = "Typing stubs for bleach"
category = "dev"
optional = false
python-versions = "*"
files = [
- {file = "types-bleach-5.0.3.1.tar.gz", hash = "sha256:ce8772ea5126dab1883851b41e3aeff229aa5213ced36096990344e632e92373"},
- {file = "types_bleach-5.0.3.1-py3-none-any.whl", hash = "sha256:af5f1b3a54ff279f54c29eccb2e6988ebb6718bc4061469588a5fd4880a79287"},
+ {file = "types-bleach-6.0.0.0.tar.gz", hash = "sha256:770ce9c7ea6173743ef1a4a70f2619bb1819bf53c7cd0336d939af93f488fbe2"},
+ {file = "types_bleach-6.0.0.0-py3-none-any.whl", hash = "sha256:75f55f035837c5fce2cd0bd5162a2a90057680a89c9275588a5c12f5f597a14a"},
]
[[package]]
@@ -2624,14 +2638,14 @@ files = [
[[package]]
name = "types-jsonschema"
-version = "4.17.0.3"
+version = "4.17.0.5"
description = "Typing stubs for jsonschema"
category = "dev"
optional = false
python-versions = "*"
files = [
- {file = "types-jsonschema-4.17.0.3.tar.gz", hash = "sha256:746aa466ffed9a1acc7bdbd0ac0b5e068f00be2ee008c1d1e14b0944a8c8b24b"},
- {file = "types_jsonschema-4.17.0.3-py3-none-any.whl", hash = "sha256:c8d5b26b7c8da6a48d7fb1ce029b97e0ff6e74db3727efb968c69f39ad013685"},
+ {file = "types-jsonschema-4.17.0.5.tar.gz", hash = "sha256:7adc7bfca4afe291de0c93eca9367aa72a4fbe8ce87fe15642c600ad97d45dd6"},
+ {file = "types_jsonschema-4.17.0.5-py3-none-any.whl", hash = "sha256:79ac8a7763fe728947af90a24168b91621edf7e8425bf3670abd4ea0d4758fba"},
]
[[package]]
@@ -2648,14 +2662,14 @@ files = [
[[package]]
name = "types-pillow"
-version = "9.4.0.10"
+version = "9.4.0.13"
description = "Typing stubs for Pillow"
category = "dev"
optional = false
python-versions = "*"
files = [
- {file = "types-Pillow-9.4.0.10.tar.gz", hash = "sha256:341c2345610bba452d1724757c7b997a60f593cf003c101ba239db003a0ae389"},
- {file = "types_Pillow-9.4.0.10-py3-none-any.whl", hash = "sha256:302ce81cfb61aacc8983a3a2ec682cbef66522a2fe8e640f648ac2e3d6f6af53"},
+ {file = "types-Pillow-9.4.0.13.tar.gz", hash = "sha256:4510aa98a28947bf63f2b29edebbd11b7cff8647d90b867cec9b3674c0a8c321"},
+ {file = "types_Pillow-9.4.0.13-py3-none-any.whl", hash = "sha256:14a8a19021b8fe569a9fef9edc64a8d8a4aef340e38669d4fb3dc05cfd941130"},
]
[[package]]
@@ -2714,14 +2728,14 @@ types-urllib3 = "<1.27"
[[package]]
name = "types-setuptools"
-version = "67.1.0.0"
+version = "67.3.0.1"
description = "Typing stubs for setuptools"
category = "dev"
optional = false
python-versions = "*"
files = [
- {file = "types-setuptools-67.1.0.0.tar.gz", hash = "sha256:162a39d22e3a5eb802197c84f16b19e798101bbd33d9437837fbb45627da5627"},
- {file = "types_setuptools-67.1.0.0-py3-none-any.whl", hash = "sha256:5bd7a10d93e468bfcb10d24cb8ea5e12ac4f4ac91267293959001f1448cf0619"},
+ {file = "types-setuptools-67.3.0.1.tar.gz", hash = "sha256:1a26d373036c720e566823b6edd664a2db4d138b6eeba856721ec1254203474f"},
+ {file = "types_setuptools-67.3.0.1-py3-none-any.whl", hash = "sha256:a7e0f0816b5b449f5bcdc0efa43da91ff81dbe6941f293a6490d68a450e130a1"},
]
[package.dependencies]
@@ -3030,4 +3044,4 @@ user-search = ["pyicu"]
[metadata]
lock-version = "2.0"
python-versions = "^3.7.1"
-content-hash = "95cb043fa56e1e3275ba7f74b68b2191bd5886eea3e06b8cd370d7fc9fea3c07"
+content-hash = "e12077711e5ff83f3c6038ea44c37bd49773799ec8245035b01094b7800c5c92"
diff --git a/pyproject.toml b/pyproject.toml
index 7f74b552c1..cef7d295c1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -89,7 +89,7 @@ manifest-path = "rust/Cargo.toml"
[tool.poetry]
name = "matrix-synapse"
-version = "1.77.0"
+version = "1.78.0rc1"
description = "Homeserver for the Matrix decentralised comms protocol"
authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
license = "Apache-2.0"
@@ -154,7 +154,9 @@ python = "^3.7.1"
# we use the TYPE_CHECKER.redefine method added in jsonschema 3.0.0
jsonschema = ">=3.0.0"
# frozendict 2.1.2 is broken on Debian 10: https://github.com/Marco-Sulla/python-frozendict/issues/41
-frozendict = ">=1,!=2.1.2"
+# We cannot test our wheels against the 2.3.5 release in CI. Putting in an upper bound for this
+# because frozendict has been more trouble than it's worth; we would like to move to immutabledict.
+frozendict = ">=1,!=2.1.2,<2.3.5"
# We require 2.1.0 or higher for type hints. Previous guard was >= 1.1.0
unpaddedbase64 = ">=2.1.0"
# We require 1.5.0 to work around an issue when running against the C implementation of
diff --git a/stubs/sortedcontainers/sortedlist.pyi b/stubs/sortedcontainers/sortedlist.pyi
index 1fe1a136f1..0e745c0a79 100644
--- a/stubs/sortedcontainers/sortedlist.pyi
+++ b/stubs/sortedcontainers/sortedlist.pyi
@@ -29,7 +29,6 @@ _Repr = Callable[[], str]
def recursive_repr(fillvalue: str = ...) -> Callable[[_Repr], _Repr]: ...
class SortedList(MutableSequence[_T]):
-
DEFAULT_LOAD_FACTOR: int = ...
def __init__(
self,
diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py
index 2b74a40166..19ca399d44 100644
--- a/synapse/_scripts/register_new_matrix_user.py
+++ b/synapse/_scripts/register_new_matrix_user.py
@@ -47,7 +47,6 @@ def request_registration(
_print: Callable[[str], None] = print,
exit: Callable[[int], None] = sys.exit,
) -> None:
-
url = "%s/_synapse/admin/v1/register" % (server_location.rstrip("/"),)
# Get the nonce
@@ -154,7 +153,6 @@ def register_new_user(
def main() -> None:
-
logging.captureWarnings(True)
parser = argparse.ArgumentParser(
diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py
index 5e137dbbf7..2c9cbf8b27 100755
--- a/synapse/_scripts/synapse_port_db.py
+++ b/synapse/_scripts/synapse_port_db.py
@@ -94,61 +94,80 @@ reactor = cast(ISynapseReactor, reactor_)
logger = logging.getLogger("synapse_port_db")
+# SQLite doesn't have a dedicated boolean type (it stores True/False as 1/0). This means
+# portdb will read sqlite bools as integers, then try to insert them into postgres
+# boolean columns---which fails. Lacking some Python-parseable metaschema, we must
+# specify which integer columns should be inserted as booleans into postgres.
BOOLEAN_COLUMNS = {
- "events": ["processed", "outlier", "contains_url"],
- "rooms": ["is_public", "has_auth_chain_index"],
+ "access_tokens": ["used"],
+ "account_validity": ["email_sent"],
+ "device_lists_changes_in_room": ["converted_to_destinations"],
+ "device_lists_outbound_pokes": ["sent"],
+ "devices": ["hidden"],
+ "e2e_fallback_keys_json": ["used"],
+ "e2e_room_keys": ["is_verified"],
"event_edges": ["is_state"],
+ "events": ["processed", "outlier", "contains_url"],
+ "local_media_repository": ["safe_from_quarantine"],
"presence_list": ["accepted"],
"presence_stream": ["currently_active"],
"public_room_list_stream": ["visibility"],
- "devices": ["hidden"],
- "device_lists_outbound_pokes": ["sent"],
- "users_who_share_rooms": ["share_private"],
- "e2e_room_keys": ["is_verified"],
- "account_validity": ["email_sent"],
+ "pushers": ["enabled"],
"redactions": ["have_censored"],
"room_stats_state": ["is_federatable"],
- "local_media_repository": ["safe_from_quarantine"],
+ "rooms": ["is_public", "has_auth_chain_index"],
"users": ["shadow_banned", "approved"],
- "e2e_fallback_keys_json": ["used"],
- "access_tokens": ["used"],
- "device_lists_changes_in_room": ["converted_to_destinations"],
- "pushers": ["enabled"],
+ "un_partial_stated_event_stream": ["rejection_status_changed"],
+ "users_who_share_rooms": ["share_private"],
}
+# These tables are never deleted from in normal operation [*], so we can resume porting
+# over rows from a previous attempt rather than starting from scratch.
+#
+# [*]: We do delete from many of these tables when purging a room, and
+# presumably when purging old events. So we might e.g.
+#
+# 1. Run portdb and port half of some table.
+# 2. Stop portdb.
+# 3. Purge something, deleting some of the rows we've ported over.
+# 4. Restart portdb. The rows deleted from sqlite are still present in postgres.
+#
+# But this isn't the end of the world: we should be able to repeat the purge
+# on the postgres DB when porting completes.
APPEND_ONLY_TABLES = [
+ "cache_invalidation_stream_by_instance",
+ "event_auth",
+ "event_edges",
+ "event_json",
"event_reference_hashes",
+ "event_search",
+ "event_to_state_groups",
"events",
- "event_json",
- "state_events",
- "room_memberships",
- "topics",
- "room_names",
- "rooms",
+ "ex_outlier_stream",
"local_media_repository",
"local_media_repository_thumbnails",
+ "presence_stream",
+ "public_room_list_stream",
+ "push_rules_stream",
+ "received_transactions",
+ "redactions",
+ "rejections",
"remote_media_cache",
"remote_media_cache_thumbnails",
- "redactions",
- "event_edges",
- "event_auth",
- "received_transactions",
+ "room_memberships",
+ "room_names",
+ "rooms",
"sent_transactions",
- "transaction_id_to_pdu",
- "users",
+ "state_events",
+ "state_group_edges",
"state_groups",
"state_groups_state",
- "event_to_state_groups",
- "rejections",
- "event_search",
- "presence_stream",
- "push_rules_stream",
- "ex_outlier_stream",
- "cache_invalidation_stream_by_instance",
- "public_room_list_stream",
- "state_group_edges",
"stream_ordering_to_exterm",
+ "topics",
+ "transaction_id_to_pdu",
+ "un_partial_stated_event_stream",
+ "users",
]
@@ -1186,7 +1205,6 @@ class CursesProgress(Progress):
if self.finished:
status = "Time spent: %s (Done!)" % (duration_str,)
else:
-
if self.total_processed > 0:
left = float(self.total_remaining) / self.total_processed
diff --git a/synapse/_scripts/synctl.py b/synapse/_scripts/synctl.py
index b4c96ad7f3..077b90935e 100755
--- a/synapse/_scripts/synctl.py
+++ b/synapse/_scripts/synctl.py
@@ -167,7 +167,6 @@ Worker = collections.namedtuple(
def main() -> None:
-
parser = argparse.ArgumentParser()
parser.add_argument(
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index a5aa2185a2..28062dd69d 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -213,7 +213,7 @@ def handle_startup_exception(e: Exception) -> NoReturn:
def redirect_stdio_to_logs() -> None:
streams = [("stdout", LogLevel.info), ("stderr", LogLevel.error)]
- for (stream, level) in streams:
+ for stream, level in streams:
oldStream = getattr(sys, stream)
loggingFile = LoggingFile(
logger=twisted.logger.Logger(namespace=stream),
diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py
index ad51f33165..b05fe2c589 100644
--- a/synapse/app/admin_cmd.py
+++ b/synapse/app/admin_cmd.py
@@ -44,6 +44,7 @@ from synapse.storage.databases.main.event_push_actions import (
)
from synapse.storage.databases.main.events_worker import EventsWorkerStore
from synapse.storage.databases.main.filtering import FilteringWorkerStore
+from synapse.storage.databases.main.media_repository import MediaRepositoryStore
from synapse.storage.databases.main.profile import ProfileWorkerStore
from synapse.storage.databases.main.push_rule import PushRulesWorkerStore
from synapse.storage.databases.main.receipts import ReceiptsWorkerStore
@@ -86,6 +87,7 @@ class AdminCmdSlavedStore(
RegistrationWorkerStore,
RoomWorkerStore,
ProfileWorkerStore,
+ MediaRepositoryStore,
):
def __init__(
self,
@@ -149,7 +151,7 @@ class FileExfiltrationWriter(ExfiltrationWriter):
with open(events_file, "a") as f:
for event in events:
- print(json.dumps(event.get_pdu_json()), file=f)
+ json.dump(event.get_pdu_json(), fp=f)
def write_state(
self, room_id: str, event_id: str, state: StateMap[EventBase]
@@ -162,7 +164,7 @@ class FileExfiltrationWriter(ExfiltrationWriter):
with open(event_file, "a") as f:
for event in state.values():
- print(json.dumps(event.get_pdu_json()), file=f)
+ json.dump(event.get_pdu_json(), fp=f)
def write_invite(
self, room_id: str, event: EventBase, state: StateMap[EventBase]
@@ -178,7 +180,7 @@ class FileExfiltrationWriter(ExfiltrationWriter):
with open(invite_state, "a") as f:
for event in state.values():
- print(json.dumps(event), file=f)
+ json.dump(event, fp=f)
def write_knock(
self, room_id: str, event: EventBase, state: StateMap[EventBase]
@@ -194,7 +196,7 @@ class FileExfiltrationWriter(ExfiltrationWriter):
with open(knock_state, "a") as f:
for event in state.values():
- print(json.dumps(event), file=f)
+ json.dump(event, fp=f)
def write_profile(self, profile: JsonDict) -> None:
user_directory = os.path.join(self.base_directory, "user_data")
@@ -202,7 +204,7 @@ class FileExfiltrationWriter(ExfiltrationWriter):
profile_file = os.path.join(user_directory, "profile")
with open(profile_file, "a") as f:
- print(json.dumps(profile), file=f)
+ json.dump(profile, fp=f)
def write_devices(self, devices: List[JsonDict]) -> None:
user_directory = os.path.join(self.base_directory, "user_data")
@@ -211,7 +213,7 @@ class FileExfiltrationWriter(ExfiltrationWriter):
for device in devices:
with open(device_file, "a") as f:
- print(json.dumps(device), file=f)
+ json.dump(device, fp=f)
def write_connections(self, connections: List[JsonDict]) -> None:
user_directory = os.path.join(self.base_directory, "user_data")
@@ -220,7 +222,7 @@ class FileExfiltrationWriter(ExfiltrationWriter):
for connection in connections:
with open(connection_file, "a") as f:
- print(json.dumps(connection), file=f)
+ json.dump(connection, fp=f)
def write_account_data(
self, file_name: str, account_data: Mapping[str, JsonDict]
@@ -233,7 +235,15 @@ class FileExfiltrationWriter(ExfiltrationWriter):
account_data_file = os.path.join(account_data_directory, file_name)
with open(account_data_file, "a") as f:
- print(json.dumps(account_data), file=f)
+ json.dump(account_data, fp=f)
+
+ def write_media_id(self, media_id: str, media_metadata: JsonDict) -> None:
+ file_directory = os.path.join(self.base_directory, "media_ids")
+ os.makedirs(file_directory, exist_ok=True)
+ media_id_file = os.path.join(file_directory, media_id)
+
+ with open(media_id_file, "w") as f:
+ json.dump(media_metadata, fp=f)
def finished(self) -> str:
return self.base_directory
diff --git a/synapse/app/complement_fork_starter.py b/synapse/app/complement_fork_starter.py
index 920538f44d..c8dc3f9d76 100644
--- a/synapse/app/complement_fork_starter.py
+++ b/synapse/app/complement_fork_starter.py
@@ -219,7 +219,7 @@ def main() -> None:
# memory space and don't need to repeat the work of loading the code!
# Instead of using fork() directly, we use the multiprocessing library,
# which uses fork() on Unix platforms.
- for (func, worker_args) in zip(worker_functions, args_by_worker):
+ for func, worker_args in zip(worker_functions, args_by_worker):
process = multiprocessing.Process(
target=_worker_entrypoint, args=(func, proxy_reactor, worker_args)
)
diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py
index 946f3a3807..0dec24369a 100644
--- a/synapse/app/generic_worker.py
+++ b/synapse/app/generic_worker.py
@@ -157,7 +157,6 @@ class GenericWorkerServer(HomeServer):
DATASTORE_CLASS = GenericWorkerSlavedStore # type: ignore
def _listen_http(self, listener_config: ListenerConfig) -> None:
-
assert listener_config.http_options is not None
# We always include a health resource.
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 6176a70eb2..b8830b1a9c 100644
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -321,7 +321,6 @@ def setup(config_options: List[str]) -> SynapseHomeServer:
and not config.registration.registrations_require_3pid
and not config.registration.registration_requires_token
):
-
raise ConfigError(
"You have enabled open registration without any verification. This is a known vector for "
"spam and abuse. If you would like to allow public registration, please consider adding email, "
diff --git a/synapse/config/consent.py b/synapse/config/consent.py
index be74609dc4..5bfd0cbb71 100644
--- a/synapse/config/consent.py
+++ b/synapse/config/consent.py
@@ -22,7 +22,6 @@ from ._base import Config
class ConsentConfig(Config):
-
section = "consent"
def __init__(self, *args: Any):
diff --git a/synapse/config/database.py b/synapse/config/database.py
index 928fec8dfe..596d8769fe 100644
--- a/synapse/config/database.py
+++ b/synapse/config/database.py
@@ -154,7 +154,6 @@ class DatabaseConfig(Config):
logger.warning(NON_SQLITE_DATABASE_PATH_WARNING)
def set_databasepath(self, database_path: str) -> None:
-
if database_path != ":memory:":
database_path = self.abspath(database_path)
diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py
index 54c91953e1..bc38fae0b6 100644
--- a/synapse/config/experimental.py
+++ b/synapse/config/experimental.py
@@ -175,8 +175,8 @@ class ExperimentalConfig(Config):
)
# MSC3873: Disambiguate event_match keys.
- self.msc3783_escape_event_match_key = experimental.get(
- "msc3783_escape_event_match_key", False
+ self.msc3873_escape_event_match_key = experimental.get(
+ "msc3873_escape_event_match_key", False
)
# MSC3952: Intentional mentions, this depends on MSC3758.
diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py
index 4d2b298a70..c205a78039 100644
--- a/synapse/config/homeserver.py
+++ b/synapse/config/homeserver.py
@@ -56,7 +56,6 @@ from .workers import WorkerConfig
class HomeServerConfig(RootConfig):
-
config_classes = [
ModulesConfig,
ServerConfig,
diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py
index 5c13fe428a..a5514e70a2 100644
--- a/synapse/config/ratelimiting.py
+++ b/synapse/config/ratelimiting.py
@@ -46,7 +46,6 @@ class RatelimitConfig(Config):
section = "ratelimiting"
def read_config(self, config: JsonDict, **kwargs: Any) -> None:
-
# Load the new-style messages config if it exists. Otherwise fall back
# to the old method.
if "rc_message" in config:
@@ -87,9 +86,18 @@ class RatelimitConfig(Config):
defaults={"per_second": 0.1, "burst_count": 5},
)
+ # It is reasonable to login with a bunch of devices at once (i.e. when
+ # setting up an account), but it is *not* valid to continually be
+ # logging into new devices.
rc_login_config = config.get("rc_login", {})
- self.rc_login_address = RatelimitSettings(rc_login_config.get("address", {}))
- self.rc_login_account = RatelimitSettings(rc_login_config.get("account", {}))
+ self.rc_login_address = RatelimitSettings(
+ rc_login_config.get("address", {}),
+ defaults={"per_second": 0.003, "burst_count": 5},
+ )
+ self.rc_login_account = RatelimitSettings(
+ rc_login_config.get("account", {}),
+ defaults={"per_second": 0.003, "burst_count": 5},
+ )
self.rc_login_failed_attempts = RatelimitSettings(
rc_login_config.get("failed_attempts", {})
)
diff --git a/synapse/config/repository.py b/synapse/config/repository.py
index e4759711ed..2da40c09f0 100644
--- a/synapse/config/repository.py
+++ b/synapse/config/repository.py
@@ -116,7 +116,6 @@ class ContentRepositoryConfig(Config):
section = "media"
def read_config(self, config: JsonDict, **kwargs: Any) -> None:
-
# Only enable the media repo if either the media repo is enabled or the
# current worker app is the media repo.
if (
diff --git a/synapse/config/server.py b/synapse/config/server.py
index ecdaa2d9dd..0e46b849cf 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -177,6 +177,7 @@ KNOWN_RESOURCES = {
"client",
"consent",
"federation",
+ "health",
"keys",
"media",
"metrics",
@@ -734,7 +735,6 @@ class ServerConfig(Config):
listeners: Optional[List[dict]],
**kwargs: Any,
) -> str:
-
_, bind_port = parse_and_validate_server_name(server_name)
if bind_port is not None:
unsecure_port = bind_port - 400
diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index 336fe3e0da..318270ebb8 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -30,7 +30,6 @@ class TlsConfig(Config):
section = "tls"
def read_config(self, config: JsonDict, **kwargs: Any) -> None:
-
self.tls_certificate_file = self.abspath(config.get("tls_certificate_path"))
self.tls_private_key_file = self.abspath(config.get("tls_private_key_path"))
diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index 86cd4af9bd..d710607c63 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -399,7 +399,7 @@ class Keyring:
# We now convert the returned list of results into a map from server
# name to key ID to FetchKeyResult, to return.
to_return: Dict[str, Dict[str, FetchKeyResult]] = {}
- for (request, results) in zip(deduped_requests, results_per_request):
+ for request, results in zip(deduped_requests, results_per_request):
to_return_by_server = to_return.setdefault(request.server_name, {})
for key_id, key_result in results.items():
existing = to_return_by_server.get(key_id)
diff --git a/synapse/events/third_party_rules.py b/synapse/events/third_party_rules.py
index 97c61cc258..9a25ed419b 100644
--- a/synapse/events/third_party_rules.py
+++ b/synapse/events/third_party_rules.py
@@ -78,7 +78,6 @@ def load_legacy_third_party_event_rules(hs: "HomeServer") -> None:
# correctly, we need to await its result. Therefore it doesn't make a lot of
# sense to make it go through the run() wrapper.
if f.__name__ == "check_event_allowed":
-
# We need to wrap check_event_allowed because its old form would return either
# a boolean or a dict, but now we want to return the dict separately from the
# boolean.
@@ -100,7 +99,6 @@ def load_legacy_third_party_event_rules(hs: "HomeServer") -> None:
return wrap_check_event_allowed
if f.__name__ == "on_create_room":
-
# We need to wrap on_create_room because its old form would return a boolean
# if the room creation is denied, but now we just want it to raise an
# exception.
diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py
index d720b5fd3f..3063df7990 100644
--- a/synapse/federation/send_queue.py
+++ b/synapse/federation/send_queue.py
@@ -314,7 +314,7 @@ class FederationRemoteSendQueue(AbstractFederationSender):
# stream position.
keyed_edus = {v: k for k, v in self.keyed_edu_changed.items()[i:j]}
- for ((destination, edu_key), pos) in keyed_edus.items():
+ for (destination, edu_key), pos in keyed_edus.items():
rows.append(
(
pos,
@@ -329,7 +329,7 @@ class FederationRemoteSendQueue(AbstractFederationSender):
j = self.edus.bisect_right(to_token) + 1
edus = self.edus.items()[i:j]
- for (pos, edu) in edus:
+ for pos, edu in edus:
rows.append((pos, EduRow(edu)))
# Sort rows based on pos
diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py
index 8b7760b2cc..b06f25b03c 100644
--- a/synapse/handlers/admin.py
+++ b/synapse/handlers/admin.py
@@ -252,16 +252,19 @@ class AdminHandler:
profile = await self.get_user(UserID.from_string(user_id))
if profile is not None:
writer.write_profile(profile)
+ logger.info("[%s] Written profile", user_id)
# Get all devices the user has
devices = await self._device_handler.get_devices_by_user(user_id)
writer.write_devices(devices)
+ logger.info("[%s] Written %s devices", user_id, len(devices))
# Get all connections the user has
connections = await self.get_whois(UserID.from_string(user_id))
writer.write_connections(
connections["devices"][""]["sessions"][0]["connections"]
)
+ logger.info("[%s] Written %s connections", user_id, len(connections))
# Get all account data the user has global and in rooms
global_data = await self._store.get_global_account_data_for_user(user_id)
@@ -269,6 +272,29 @@ class AdminHandler:
writer.write_account_data("global", global_data)
for room_id in by_room_data:
writer.write_account_data(room_id, by_room_data[room_id])
+ logger.info(
+ "[%s] Written account data for %s rooms", user_id, len(by_room_data)
+ )
+
+ # Get all media ids the user has
+ limit = 100
+ start = 0
+ while True:
+ media_ids, total = await self._store.get_local_media_by_user_paginate(
+ start, limit, user_id
+ )
+ for media in media_ids:
+ writer.write_media_id(media["media_id"], media)
+
+ logger.info(
+ "[%s] Written %d media_ids of %s",
+ user_id,
+ (start + len(media_ids)),
+ total,
+ )
+ if (start + limit) >= total:
+ break
+ start += limit
return writer.finished()
@@ -360,6 +386,18 @@ class ExfiltrationWriter(metaclass=abc.ABCMeta):
raise NotImplementedError()
@abc.abstractmethod
+ def write_media_id(self, media_id: str, media_metadata: JsonDict) -> None:
+ """Write the media's metadata of a user.
+ Exports only the metadata, as this can be fetched from the database via
+ read only. In order to access the files, a connection to the correct
+ media repository would be required.
+
+ Args:
+ media_id: ID of the media.
+ media_metadata: Metadata of one media file.
+ """
+
+ @abc.abstractmethod
def finished(self) -> Any:
"""Called when all data has successfully been exported and written.
diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index 5d1d21cdc8..ec3ab968e9 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -737,7 +737,7 @@ class ApplicationServicesHandler:
)
ret = []
- for (success, result) in results:
+ for success, result in results:
if success:
ret.extend(result)
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index cf12b55d21..b12bc4c9a3 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -815,7 +815,6 @@ class AuthHandler:
now_ms = self._clock.time_msec()
if existing_token.expiry_ts is not None and existing_token.expiry_ts < now_ms:
-
raise SynapseError(
HTTPStatus.FORBIDDEN,
"The supplied refresh token has expired",
@@ -2259,7 +2258,6 @@ class PasswordAuthProvider:
async def on_logged_out(
self, user_id: str, device_id: Optional[str], access_token: str
) -> None:
-
# call all of the on_logged_out callbacks
for callback in self.on_logged_out_callbacks:
try:
diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py
index a5798e9483..1fb23cc9bf 100644
--- a/synapse/handlers/directory.py
+++ b/synapse/handlers/directory.py
@@ -497,9 +497,11 @@ class DirectoryHandler:
raise SynapseError(403, "Not allowed to publish room")
# Check if publishing is blocked by a third party module
- allowed_by_third_party_rules = await (
- self.third_party_event_rules.check_visibility_can_be_modified(
- room_id, visibility
+ allowed_by_third_party_rules = (
+ await (
+ self.third_party_event_rules.check_visibility_can_be_modified(
+ room_id, visibility
+ )
)
)
if not allowed_by_third_party_rules:
diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index 83f53ceb88..50317ec753 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -188,7 +188,6 @@ class E2eRoomKeysHandler:
# XXX: perhaps we should use a finer grained lock here?
async with self._upload_linearizer.queue(user_id):
-
# Check that the version we're trying to upload is the current version
try:
version_info = await self.store.get_e2e_room_keys_version_info(user_id)
diff --git a/synapse/handlers/event_auth.py b/synapse/handlers/event_auth.py
index 46dd63c3f0..c508861b6a 100644
--- a/synapse/handlers/event_auth.py
+++ b/synapse/handlers/event_auth.py
@@ -236,7 +236,6 @@ class EventAuthHandler:
# in any of them.
allowed_rooms = await self.get_rooms_that_allow_join(state_ids)
if not await self.is_user_in_rooms(allowed_rooms, user_id):
-
# If this is a remote request, the user might be in an allowed room
# that we do not know about.
if get_domain_from_id(user_id) != self._server_name:
diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py
index 1a29abde98..aead0b44b9 100644
--- a/synapse/handlers/initial_sync.py
+++ b/synapse/handlers/initial_sync.py
@@ -124,7 +124,6 @@ class InitialSyncHandler:
as_client_event: bool = True,
include_archived: bool = False,
) -> JsonDict:
-
memberships = [Membership.INVITE, Membership.JOIN]
if include_archived:
memberships.append(Membership.LEAVE)
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 87af31aa27..4ad2233573 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -777,7 +777,6 @@ class PresenceHandler(BasePresenceHandler):
)
if self.unpersisted_users_changes:
-
await self.store.update_presence(
[
self.user_to_current_state[user_id]
@@ -823,7 +822,6 @@ class PresenceHandler(BasePresenceHandler):
now = self.clock.time_msec()
with Measure(self.clock, "presence_update_states"):
-
# NOTE: We purposefully don't await between now and when we've
# calculated what we want to do with the new states, to avoid races.
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index c611efb760..e4e506e62c 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -476,7 +476,7 @@ class RegistrationHandler:
# create room expects the localpart of the room alias
config["room_alias_name"] = room_alias.localpart
- info, _ = await room_creation_handler.create_room(
+ room_id, _, _ = await room_creation_handler.create_room(
fake_requester,
config=config,
ratelimit=False,
@@ -490,7 +490,7 @@ class RegistrationHandler:
user_id, authenticated_entity=self._server_name
),
target=UserID.from_string(user_id),
- room_id=info["room_id"],
+ room_id=room_id,
# Since it was just created, there are no remote hosts.
remote_room_hosts=[],
action="join",
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 837dabb3b7..a26ec02284 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -690,13 +690,14 @@ class RoomCreationHandler:
config: JsonDict,
ratelimit: bool = True,
creator_join_profile: Optional[JsonDict] = None,
- ) -> Tuple[dict, int]:
+ ) -> Tuple[str, Optional[RoomAlias], int]:
"""Creates a new room.
Args:
- requester:
- The user who requested the room creation.
- config : A dict of configuration options.
+ requester: The user who requested the room creation.
+ config: A dict of configuration options. This will be the body of
+ a /createRoom request; see
+ https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3createroom
ratelimit: set to False to disable the rate limiter
creator_join_profile:
@@ -707,14 +708,17 @@ class RoomCreationHandler:
`avatar_url` and/or `displayname`.
Returns:
- First, a dict containing the keys `room_id` and, if an alias
- was, requested, `room_alias`. Secondly, the stream_id of the
- last persisted event.
+ A 3-tuple containing:
+ - the room ID;
+ - if requested, the room alias, otherwise None; and
+ - the `stream_id` of the last persisted event.
Raises:
- SynapseError if the room ID couldn't be stored, 3pid invitation config
- validation failed, or something went horribly wrong.
- ResourceLimitError if server is blocked to some resource being
- exceeded
+ SynapseError:
+ if the room ID couldn't be stored, 3pid invitation config
+ validation failed, or something went horribly wrong.
+ ResourceLimitError:
+ if server is blocked to some resource being
+ exceeded
"""
user_id = requester.user.to_string()
@@ -864,9 +868,11 @@ class RoomCreationHandler:
)
# Check whether this visibility value is blocked by a third party module
- allowed_by_third_party_rules = await (
- self.third_party_event_rules.check_visibility_can_be_modified(
- room_id, visibility
+ allowed_by_third_party_rules = (
+ await (
+ self.third_party_event_rules.check_visibility_can_be_modified(
+ room_id, visibility
+ )
)
)
if not allowed_by_third_party_rules:
@@ -1024,11 +1030,6 @@ class RoomCreationHandler:
last_sent_event_id = member_event_id
depth += 1
- result = {"room_id": room_id}
-
- if room_alias:
- result["room_alias"] = room_alias.to_string()
-
# Always wait for room creation to propagate before returning
await self._replication.wait_for_stream_position(
self.hs.config.worker.events_shard_config.get_instance(room_id),
@@ -1036,7 +1037,7 @@ class RoomCreationHandler:
last_stream_id,
)
- return result, last_stream_id
+ return room_id, room_alias, last_stream_id
async def _send_events_for_new_room(
self,
@@ -1825,7 +1826,7 @@ class RoomShutdownHandler:
new_room_user_id, authenticated_entity=requester_user_id
)
- info, stream_id = await self._room_creation_handler.create_room(
+ new_room_id, _, stream_id = await self._room_creation_handler.create_room(
room_creator_requester,
config={
"preset": RoomCreationPreset.PUBLIC_CHAT,
@@ -1834,7 +1835,6 @@ class RoomShutdownHandler:
},
ratelimit=False,
)
- new_room_id = info["room_id"]
logger.info(
"Shutting down room %r, joining to new room: %r", room_id, new_room_id
@@ -1887,6 +1887,7 @@ class RoomShutdownHandler:
# Join users to new room
if new_room_user_id:
+ assert new_room_id is not None
await self.room_member_handler.update_membership(
requester=target_requester,
target=target_requester.user,
@@ -1919,6 +1920,7 @@ class RoomShutdownHandler:
aliases_for_room = await self.store.get_aliases_for_room(room_id)
+ assert new_room_id is not None
await self.store.update_aliases_for_room(
room_id, new_room_id, requester_user_id
)
diff --git a/synapse/handlers/room_batch.py b/synapse/handlers/room_batch.py
index c73d2adaad..5d4ca0e2d2 100644
--- a/synapse/handlers/room_batch.py
+++ b/synapse/handlers/room_batch.py
@@ -374,7 +374,7 @@ class RoomBatchHandler:
# correct stream_ordering as they are backfilled (which decrements).
# Events are sorted by (topological_ordering, stream_ordering)
# where topological_ordering is just depth.
- for (event, context) in reversed(events_to_persist):
+ for event, context in reversed(events_to_persist):
# This call can't raise `PartialStateConflictError` since we forbid
# use of the historical batch API during partial state
await self.event_creation_handler.handle_new_client_event(
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 4e4595312c..fd6d946c37 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -1297,7 +1297,6 @@ class SyncHandler:
return RoomNotifCounts.empty()
with Measure(self.clock, "unread_notifs_for_room_id"):
-
return await self.store.get_unread_event_push_actions_by_room_for_user(
room_id,
sync_config.user.to_string(),
diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py
index 5aed71262f..c70eee649c 100644
--- a/synapse/logging/opentracing.py
+++ b/synapse/logging/opentracing.py
@@ -524,6 +524,7 @@ def whitelisted_homeserver(destination: str) -> bool:
# Start spans and scopes
+
# Could use kwargs but I want these to be explicit
def start_active_span(
operation_name: str,
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index b01372565d..8ce5887229 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -87,7 +87,6 @@ class LaterGauge(Collector):
]
def collect(self) -> Iterable[Metric]:
-
g = GaugeMetricFamily(self.name, self.desc, labels=self.labels)
try:
diff --git a/synapse/metrics/_gc.py b/synapse/metrics/_gc.py
index b7d47ce3e7..a22c4e5bbd 100644
--- a/synapse/metrics/_gc.py
+++ b/synapse/metrics/_gc.py
@@ -139,7 +139,6 @@ def install_gc_manager() -> None:
class PyPyGCStats(Collector):
def collect(self) -> Iterable[Metric]:
-
# @stats is a pretty-printer object with __str__() returning a nice table,
# plus some fields that contain data from that table.
# unfortunately, fields are pretty-printed themselves (i. e. '4.5MB').
diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py
index d22dd19d38..1964276a54 100644
--- a/synapse/module_api/__init__.py
+++ b/synapse/module_api/__init__.py
@@ -1576,14 +1576,14 @@ class ModuleApi:
)
requester = create_requester(user_id)
- room_id_and_alias, _ = await self._hs.get_room_creation_handler().create_room(
+ room_id, room_alias, _ = await self._hs.get_room_creation_handler().create_room(
requester=requester,
config=config,
ratelimit=ratelimit,
creator_join_profile=creator_join_profile,
)
-
- return room_id_and_alias["room_id"], room_id_and_alias.get("room_alias", None)
+ room_alias_str = room_alias.to_string() if room_alias else None
+ return room_id, room_alias_str
async def set_displayname(
self,
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 5fc38431ba..3c4a152d6b 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -276,7 +276,7 @@ class BulkPushRuleEvaluator:
if related_event is not None:
related_events[relation_type] = _flatten_dict(
related_event,
- msc3783_escape_event_match_key=self.hs.config.experimental.msc3783_escape_event_match_key,
+ msc3873_escape_event_match_key=self.hs.config.experimental.msc3873_escape_event_match_key,
)
reply_event_id = (
@@ -294,7 +294,7 @@ class BulkPushRuleEvaluator:
if related_event is not None:
related_events["m.in_reply_to"] = _flatten_dict(
related_event,
- msc3783_escape_event_match_key=self.hs.config.experimental.msc3783_escape_event_match_key,
+ msc3873_escape_event_match_key=self.hs.config.experimental.msc3873_escape_event_match_key,
)
# indicate that this is from a fallback relation.
@@ -330,7 +330,6 @@ class BulkPushRuleEvaluator:
context: EventContext,
event_id_to_event: Mapping[str, EventBase],
) -> None:
-
if (
not event.internal_metadata.is_notifiable()
or event.internal_metadata.is_historical()
@@ -413,7 +412,7 @@ class BulkPushRuleEvaluator:
evaluator = PushRuleEvaluator(
_flatten_dict(
event,
- msc3783_escape_event_match_key=self.hs.config.experimental.msc3783_escape_event_match_key,
+ msc3873_escape_event_match_key=self.hs.config.experimental.msc3873_escape_event_match_key,
),
has_mentions,
user_mentions,
@@ -508,7 +507,7 @@ def _flatten_dict(
prefix: Optional[List[str]] = None,
result: Optional[Dict[str, JsonValue]] = None,
*,
- msc3783_escape_event_match_key: bool = False,
+ msc3873_escape_event_match_key: bool = False,
) -> Dict[str, JsonValue]:
"""
Given a JSON dictionary (or event) which might contain sub dictionaries,
@@ -537,7 +536,7 @@ def _flatten_dict(
if result is None:
result = {}
for key, value in d.items():
- if msc3783_escape_event_match_key:
+ if msc3873_escape_event_match_key:
# Escape periods in the key with a backslash (and backslashes with an
# extra backslash). This is since a period is used as a separator between
# nested fields.
@@ -553,7 +552,7 @@ def _flatten_dict(
value,
prefix=(prefix + [key]),
result=result,
- msc3783_escape_event_match_key=msc3783_escape_event_match_key,
+ msc3873_escape_event_match_key=msc3873_escape_event_match_key,
)
# `room_version` should only ever be set when looking at the top level of an event
diff --git a/synapse/replication/http/account_data.py b/synapse/replication/http/account_data.py
index 2374f810c9..111ec07e64 100644
--- a/synapse/replication/http/account_data.py
+++ b/synapse/replication/http/account_data.py
@@ -265,7 +265,6 @@ class ReplicationRemoveTagRestServlet(ReplicationEndpoint):
@staticmethod
async def _serialize_payload(user_id: str, room_id: str, tag: str) -> JsonDict: # type: ignore[override]
-
return {}
async def _handle_request( # type: ignore[override]
diff --git a/synapse/replication/http/devices.py b/synapse/replication/http/devices.py
index ecea6fc915..cc3929dcf5 100644
--- a/synapse/replication/http/devices.py
+++ b/synapse/replication/http/devices.py
@@ -195,7 +195,6 @@ class ReplicationUploadKeysForUserRestServlet(ReplicationEndpoint):
async def _serialize_payload( # type: ignore[override]
user_id: str, device_id: str, keys: JsonDict
) -> JsonDict:
-
return {
"user_id": user_id,
"device_id": device_id,
diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py
index cc0528bd8e..424854efbe 100644
--- a/synapse/replication/tcp/client.py
+++ b/synapse/replication/tcp/client.py
@@ -370,15 +370,23 @@ class ReplicationDataHandler:
# We measure here to get in flight counts and average waiting time.
with Measure(self._clock, "repl.wait_for_stream_position"):
logger.info(
- "Waiting for repl stream %r to reach %s (%s)",
+ "Waiting for repl stream %r to reach %s (%s); currently at: %s",
stream_name,
position,
instance_name,
+ current_position,
)
try:
await make_deferred_yieldable(deferred)
except defer.TimeoutError:
- logger.error("Timed out waiting for stream %s", stream_name)
+ logger.error(
+ "Timed out waiting for repl stream %r to reach %s (%s)"
+ "; currently at: %s",
+ stream_name,
+ position,
+ instance_name,
+ self._streams[stream_name].current_token(instance_name),
+ )
return
logger.info(
diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py
index fd1c0ec6af..dfc061eb5e 100644
--- a/synapse/replication/tcp/redis.py
+++ b/synapse/replication/tcp/redis.py
@@ -328,7 +328,6 @@ class RedisDirectTcpReplicationClientFactory(SynapseRedisFactory):
outbound_redis_connection: txredisapi.ConnectionHandler,
channel_names: List[str],
):
-
super().__init__(
hs,
uuid="subscriber",
diff --git a/synapse/replication/tcp/streams/events.py b/synapse/replication/tcp/streams/events.py
index 14b6705862..ad9b760713 100644
--- a/synapse/replication/tcp/streams/events.py
+++ b/synapse/replication/tcp/streams/events.py
@@ -139,7 +139,6 @@ class EventsStream(Stream):
current_token: Token,
target_row_count: int,
) -> StreamUpdateResult:
-
# the events stream merges together three separate sources:
# * new events
# * current_state changes
diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py
index 1d6e4982d7..4de56bf13f 100644
--- a/synapse/rest/admin/rooms.py
+++ b/synapse/rest/admin/rooms.py
@@ -75,7 +75,6 @@ class RoomRestV2Servlet(RestServlet):
async def on_DELETE(
self, request: SynapseRequest, room_id: str
) -> Tuple[int, JsonDict]:
-
requester = await self._auth.get_user_by_req(request)
await assert_user_is_admin(self._auth, requester)
@@ -144,7 +143,6 @@ class DeleteRoomStatusByRoomIdRestServlet(RestServlet):
async def on_GET(
self, request: SynapseRequest, room_id: str
) -> Tuple[int, JsonDict]:
-
await assert_requester_is_admin(self._auth, request)
if not RoomID.is_valid(room_id):
@@ -181,7 +179,6 @@ class DeleteRoomStatusByDeleteIdRestServlet(RestServlet):
async def on_GET(
self, request: SynapseRequest, delete_id: str
) -> Tuple[int, JsonDict]:
-
await assert_requester_is_admin(self._auth, request)
delete_status = self._pagination_handler.get_delete_status(delete_id)
@@ -438,7 +435,6 @@ class RoomStateRestServlet(RestServlet):
class JoinRoomAliasServlet(ResolveRoomIdMixin, RestServlet):
-
PATTERNS = admin_patterns("/join/(?P<room_identifier>[^/]*)$")
def __init__(self, hs: "HomeServer"):
diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py
index 0c0bf540b9..7cc4db20d6 100644
--- a/synapse/rest/admin/users.py
+++ b/synapse/rest/admin/users.py
@@ -683,8 +683,12 @@ class AccountValidityRenewServlet(RestServlet):
await assert_requester_is_admin(self.auth, request)
if self.account_activity_handler.on_legacy_admin_request_callback:
- expiration_ts = await (
- self.account_activity_handler.on_legacy_admin_request_callback(request)
+ expiration_ts = (
+ await (
+ self.account_activity_handler.on_legacy_admin_request_callback(
+ request
+ )
+ )
)
else:
body = parse_json_object_from_request(request)
diff --git a/synapse/rest/client/auth.py b/synapse/rest/client/auth.py
index eb77337044..276a1b405d 100644
--- a/synapse/rest/client/auth.py
+++ b/synapse/rest/client/auth.py
@@ -97,7 +97,6 @@ class AuthRestServlet(RestServlet):
return None
async def on_POST(self, request: Request, stagetype: str) -> None:
-
session = parse_string(request, "session")
if not session:
raise SynapseError(400, "No session supplied")
diff --git a/synapse/rest/client/filter.py b/synapse/rest/client/filter.py
index cc1c2f9731..236199897c 100644
--- a/synapse/rest/client/filter.py
+++ b/synapse/rest/client/filter.py
@@ -79,7 +79,6 @@ class CreateFilterRestServlet(RestServlet):
async def on_POST(
self, request: SynapseRequest, user_id: str
) -> Tuple[int, JsonDict]:
-
target_user = UserID.from_string(user_id)
requester = await self.auth.get_user_by_req(request)
diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py
index 3cb1e7e375..bce806f2bb 100644
--- a/synapse/rest/client/register.py
+++ b/synapse/rest/client/register.py
@@ -628,10 +628,12 @@ class RegisterRestServlet(RestServlet):
if not password_hash:
raise SynapseError(400, "Missing params: password", Codes.MISSING_PARAM)
- desired_username = await (
- self.password_auth_provider.get_username_for_registration(
- auth_result,
- params,
+ desired_username = (
+ await (
+ self.password_auth_provider.get_username_for_registration(
+ auth_result,
+ params,
+ )
)
)
@@ -682,9 +684,11 @@ class RegisterRestServlet(RestServlet):
session_id
)
- display_name = await (
- self.password_auth_provider.get_displayname_for_registration(
- auth_result, params
+ display_name = (
+ await (
+ self.password_auth_provider.get_displayname_for_registration(
+ auth_result, params
+ )
)
)
diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py
index d0db85cca7..14b04810a1 100644
--- a/synapse/rest/client/room.py
+++ b/synapse/rest/client/room.py
@@ -160,11 +160,11 @@ class RoomCreateRestServlet(TransactionRestServlet):
async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
requester = await self.auth.get_user_by_req(request)
- info, _ = await self._room_creation_handler.create_room(
+ room_id, _, _ = await self._room_creation_handler.create_room(
requester, self.get_room_config(request)
)
- return 200, info
+ return 200, {"room_id": room_id}
def get_room_config(self, request: Request) -> JsonDict:
user_supplied_config = parse_json_object_from_request(request)
diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py
index 6e035afcce..ef8334ae25 100644
--- a/synapse/rest/media/v1/_base.py
+++ b/synapse/rest/media/v1/_base.py
@@ -270,7 +270,6 @@ async def respond_with_responder(
logger.debug("Responding to media request with responder %s", responder)
add_file_headers(request, media_type, file_size, upload_name)
try:
-
await responder.write_to_consumer(request)
except Exception as e:
# The majority of the time this will be due to the client having gone
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index a8f6fd6b35..4a594ab9d8 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -163,6 +163,10 @@ class PreviewUrlResource(DirectServeJsonResource):
7. Stores the result in the database cache.
4. Returns the result.
+ If any additional requests (e.g. from oEmbed autodiscovery, step 5.3 or
+ image thumbnailing, step 5.4 or 6.4) fails then the URL preview as a whole
+ does not fail. As much information as possible is returned.
+
The in-memory cache expires after 1 hour.
Expired entries in the database cache (and their associated media files) are
@@ -364,16 +368,25 @@ class PreviewUrlResource(DirectServeJsonResource):
oembed_url = self._oembed.autodiscover_from_html(tree)
og_from_oembed: JsonDict = {}
if oembed_url:
- oembed_info = await self._handle_url(
- oembed_url, user, allow_data_urls=True
- )
- (
- og_from_oembed,
- author_name,
- expiration_ms,
- ) = await self._handle_oembed_response(
- url, oembed_info, expiration_ms
- )
+ try:
+ oembed_info = await self._handle_url(
+ oembed_url, user, allow_data_urls=True
+ )
+ except Exception as e:
+ # Fetching the oEmbed info failed, don't block the entire URL preview.
+ logger.warning(
+ "oEmbed fetch failed during URL preview: %s errored with %s",
+ oembed_url,
+ e,
+ )
+ else:
+ (
+ og_from_oembed,
+ author_name,
+ expiration_ms,
+ ) = await self._handle_oembed_response(
+ url, oembed_info, expiration_ms
+ )
# Parse Open Graph information from the HTML in case the oEmbed
# response failed or is incomplete.
diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py
index 5f725c7600..3e720018b3 100644
--- a/synapse/rest/media/v1/thumbnail_resource.py
+++ b/synapse/rest/media/v1/thumbnail_resource.py
@@ -69,7 +69,8 @@ class ThumbnailResource(DirectServeJsonResource):
width = parse_integer(request, "width", required=True)
height = parse_integer(request, "height", required=True)
method = parse_string(request, "method", "scale")
- m_type = parse_string(request, "type", "image/png")
+ # TODO Parse the Accept header to get an prioritised list of thumbnail types.
+ m_type = "image/png"
if server_name == self.server_name:
if self.dynamic_thumbnails:
diff --git a/synapse/rest/media/v1/thumbnailer.py b/synapse/rest/media/v1/thumbnailer.py
index 9480cc5763..f909a4fb9a 100644
--- a/synapse/rest/media/v1/thumbnailer.py
+++ b/synapse/rest/media/v1/thumbnailer.py
@@ -38,7 +38,6 @@ class ThumbnailError(Exception):
class Thumbnailer:
-
FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG"}
@staticmethod
diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py
index 564e3705c2..9732dbdb6e 100644
--- a/synapse/server_notices/server_notices_manager.py
+++ b/synapse/server_notices/server_notices_manager.py
@@ -178,7 +178,7 @@ class ServerNoticesManager:
"avatar_url": self._config.servernotices.server_notices_mxid_avatar_url,
}
- info, _ = await self._room_creation_handler.create_room(
+ room_id, _, _ = await self._room_creation_handler.create_room(
requester,
config={
"preset": RoomCreationPreset.PRIVATE_CHAT,
@@ -188,7 +188,6 @@ class ServerNoticesManager:
ratelimit=False,
creator_join_profile=join_profile,
)
- room_id = info["room_id"]
self.maybe_get_notice_room_for_user.invalidate((user_id,))
diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py
index 8e61aba454..0d75d9739a 100644
--- a/synapse/storage/databases/main/deviceinbox.py
+++ b/synapse/storage/databases/main/deviceinbox.py
@@ -721,8 +721,8 @@ class DeviceInboxWorkerStore(SQLBaseStore):
],
)
- for (user_id, messages_by_device) in edu["messages"].items():
- for (device_id, msg) in messages_by_device.items():
+ for user_id, messages_by_device in edu["messages"].items():
+ for device_id, msg in messages_by_device.items():
with start_active_span("store_outgoing_to_device_message"):
set_tag(SynapseTags.TO_DEVICE_EDU_ID, edu["sender"])
set_tag(SynapseTags.TO_DEVICE_EDU_ID, edu["message_id"])
@@ -959,7 +959,6 @@ class DeviceInboxBackgroundUpdateStore(SQLBaseStore):
def _remove_dead_devices_from_device_inbox_txn(
txn: LoggingTransaction,
) -> Tuple[int, bool]:
-
if "max_stream_id" in progress:
max_stream_id = progress["max_stream_id"]
else:
diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py
index 1ca66d57d4..0dd15f16ff 100644
--- a/synapse/storage/databases/main/devices.py
+++ b/synapse/storage/databases/main/devices.py
@@ -512,7 +512,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore):
results.append(("org.matrix.signing_key_update", result))
if issue_8631_logger.isEnabledFor(logging.DEBUG):
- for (user_id, edu) in results:
+ for user_id, edu in results:
issue_8631_logger.debug(
"device update to %s for %s from %s to %s: %s",
destination,
@@ -1316,7 +1316,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore):
)
"""
count = 0
- for (destination, user_id, stream_id, device_id) in rows:
+ for destination, user_id, stream_id, device_id in rows:
txn.execute(
delete_sql, (destination, user_id, stream_id, stream_id, device_id)
)
diff --git a/synapse/storage/databases/main/e2e_room_keys.py b/synapse/storage/databases/main/e2e_room_keys.py
index 6240f9a75e..9f8d2e4bea 100644
--- a/synapse/storage/databases/main/e2e_room_keys.py
+++ b/synapse/storage/databases/main/e2e_room_keys.py
@@ -108,7 +108,7 @@ class EndToEndRoomKeyStore(SQLBaseStore):
raise StoreError(404, "No backup with that version exists")
values = []
- for (room_id, session_id, room_key) in room_keys:
+ for room_id, session_id, room_key in room_keys:
values.append(
(
user_id,
diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py
index 2c2d145666..b9c39b1718 100644
--- a/synapse/storage/databases/main/end_to_end_keys.py
+++ b/synapse/storage/databases/main/end_to_end_keys.py
@@ -268,7 +268,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker
)
# add each cross-signing signature to the correct device in the result dict.
- for (user_id, key_id, device_id, signature) in cross_sigs_result:
+ for user_id, key_id, device_id, signature in cross_sigs_result:
target_device_result = result[user_id][device_id]
# We've only looked up cross-signatures for non-deleted devices with key
# data.
@@ -311,7 +311,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker
# devices.
user_list = []
user_device_list = []
- for (user_id, device_id) in query_list:
+ for user_id, device_id in query_list:
if device_id is None:
user_list.append(user_id)
else:
@@ -353,7 +353,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker
txn.execute(sql, query_params)
- for (user_id, device_id, display_name, key_json) in txn:
+ for user_id, device_id, display_name, key_json in txn:
assert device_id is not None
if include_deleted_devices:
deleted_devices.remove((user_id, device_id))
@@ -382,7 +382,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker
signature_query_clauses = []
signature_query_params = []
- for (user_id, device_id) in device_query:
+ for user_id, device_id in device_query:
signature_query_clauses.append(
"target_user_id = ? AND target_device_id = ? AND user_id = ?"
)
diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py
index ca780cca36..ff3edeb716 100644
--- a/synapse/storage/databases/main/event_federation.py
+++ b/synapse/storage/databases/main/event_federation.py
@@ -1612,7 +1612,6 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
latest_events: List[str],
limit: int,
) -> List[str]:
-
seen_events = set(earliest_events)
front = set(latest_events) - seen_events
event_results: List[str] = []
diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py
index 7996cbb557..73b8aea16c 100644
--- a/synapse/storage/databases/main/events.py
+++ b/synapse/storage/databases/main/events.py
@@ -469,7 +469,6 @@ class PersistEventsStore:
txn: LoggingTransaction,
events: List[EventBase],
) -> None:
-
# We only care about state events, so this if there are no state events.
if not any(e.is_state() for e in events):
return
diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py
index 584536111d..0a275e6ce6 100644
--- a/synapse/storage/databases/main/events_bg_updates.py
+++ b/synapse/storage/databases/main/events_bg_updates.py
@@ -709,7 +709,7 @@ class EventsBackgroundUpdatesStore(SQLBaseStore):
nbrows = 0
last_row_event_id = ""
- for (event_id, event_json_raw) in results:
+ for event_id, event_json_raw in results:
try:
event_json = db_to_json(event_json_raw)
@@ -1167,7 +1167,7 @@ class EventsBackgroundUpdatesStore(SQLBaseStore):
results = list(txn)
# (event_id, parent_id, rel_type) for each relation
relations_to_insert: List[Tuple[str, str, str]] = []
- for (event_id, event_json_raw) in results:
+ for event_id, event_json_raw in results:
try:
event_json = db_to_json(event_json_raw)
except Exception as e:
diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py
index 6d0ef10258..b7e7498125 100644
--- a/synapse/storage/databases/main/events_worker.py
+++ b/synapse/storage/databases/main/events_worker.py
@@ -1493,7 +1493,7 @@ class EventsWorkerStore(SQLBaseStore):
txn.execute(redactions_sql + clause, args)
- for (redacter, redacted) in txn:
+ for redacter, redacted in txn:
d = event_dict.get(redacted)
if d:
d.redactions.append(redacter)
diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py
index b202c5eb87..fa8be214ce 100644
--- a/synapse/storage/databases/main/media_repository.py
+++ b/synapse/storage/databases/main/media_repository.py
@@ -196,7 +196,6 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore):
def get_local_media_by_user_paginate_txn(
txn: LoggingTransaction,
) -> Tuple[List[Dict[str, Any]], int]:
-
# Set ordering
order_by_column = MediaSortOrder(order_by).value
diff --git a/synapse/storage/databases/main/pusher.py b/synapse/storage/databases/main/pusher.py
index df53e726e6..fddbc07afa 100644
--- a/synapse/storage/databases/main/pusher.py
+++ b/synapse/storage/databases/main/pusher.py
@@ -344,7 +344,6 @@ class PusherWorkerStore(SQLBaseStore):
last_user = progress.get("last_user", "")
def _delete_pushers(txn: LoggingTransaction) -> int:
-
sql = """
SELECT name FROM users
WHERE deactivated = ? and name > ?
@@ -392,7 +391,6 @@ class PusherWorkerStore(SQLBaseStore):
last_pusher = progress.get("last_pusher", 0)
def _delete_pushers(txn: LoggingTransaction) -> int:
-
sql = """
SELECT p.id, access_token FROM pushers AS p
LEFT JOIN access_tokens AS a ON (p.access_token = a.id)
@@ -449,7 +447,6 @@ class PusherWorkerStore(SQLBaseStore):
last_pusher = progress.get("last_pusher", 0)
def _delete_pushers(txn: LoggingTransaction) -> int:
-
sql = """
SELECT p.id, p.user_name, p.app_id, p.pushkey
FROM pushers AS p
diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py
index dddf49c2d5..92a82240ab 100644
--- a/synapse/storage/databases/main/receipts.py
+++ b/synapse/storage/databases/main/receipts.py
@@ -887,7 +887,6 @@ class ReceiptsBackgroundUpdateStore(SQLBaseStore):
def _populate_receipt_event_stream_ordering_txn(
txn: LoggingTransaction,
) -> bool:
-
if "max_stream_id" in progress:
max_stream_id = progress["max_stream_id"]
else:
diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py
index 644bbb8878..39f89291b2 100644
--- a/synapse/storage/databases/main/room.py
+++ b/synapse/storage/databases/main/room.py
@@ -2168,7 +2168,6 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore):
def _get_event_report_txn(
txn: LoggingTransaction, report_id: int
) -> Optional[Dict[str, Any]]:
-
sql = """
SELECT
er.id,
diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py
index 3fe433f66c..a7aae661d8 100644
--- a/synapse/storage/databases/main/search.py
+++ b/synapse/storage/databases/main/search.py
@@ -122,7 +122,6 @@ class SearchWorkerStore(SQLBaseStore):
class SearchBackgroundUpdateStore(SearchWorkerStore):
-
EVENT_SEARCH_UPDATE_NAME = "event_search"
EVENT_SEARCH_ORDER_UPDATE_NAME = "event_search_order"
EVENT_SEARCH_USE_GIN_POSTGRES_NAME = "event_search_postgres_gin"
@@ -615,7 +614,6 @@ class SearchStore(SearchBackgroundUpdateStore):
"""
count_args = [search_query] + count_args
elif isinstance(self.database_engine, Sqlite3Engine):
-
# We use CROSS JOIN here to ensure we use the right indexes.
# https://sqlite.org/optoverview.html#crossjoin
#
diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py
index ba325d390b..ebb2ae964f 100644
--- a/synapse/storage/databases/main/state.py
+++ b/synapse/storage/databases/main/state.py
@@ -490,7 +490,6 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
class MainStateBackgroundUpdateStore(RoomMemberWorkerStore):
-
CURRENT_STATE_INDEX_UPDATE_NAME = "current_state_members_idx"
EVENT_STATE_GROUP_INDEX_UPDATE_NAME = "event_to_state_groups_sg_index"
DELETE_CURRENT_STATE_UPDATE_NAME = "delete_old_current_state_events"
diff --git a/synapse/storage/databases/main/stats.py b/synapse/storage/databases/main/stats.py
index d7b7d0c3c9..d3393d8e49 100644
--- a/synapse/storage/databases/main/stats.py
+++ b/synapse/storage/databases/main/stats.py
@@ -461,7 +461,7 @@ class StatsStore(StateDeltasStore):
insert_cols = []
qargs = []
- for (key, val) in chain(
+ for key, val in chain(
keyvalues.items(), absolutes.items(), additive_relatives.items()
):
insert_cols.append(key)
diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py
index 818c46182e..ac5fbf6b86 100644
--- a/synapse/storage/databases/main/stream.py
+++ b/synapse/storage/databases/main/stream.py
@@ -87,6 +87,7 @@ MAX_STREAM_SIZE = 1000
_STREAM_TOKEN = "stream"
_TOPOLOGICAL_TOKEN = "topological"
+
# Used as return values for pagination APIs
@attr.s(slots=True, frozen=True, auto_attribs=True)
class _EventDictReturn:
diff --git a/synapse/storage/databases/main/transactions.py b/synapse/storage/databases/main/transactions.py
index 6b33d809b6..6d72bd9f67 100644
--- a/synapse/storage/databases/main/transactions.py
+++ b/synapse/storage/databases/main/transactions.py
@@ -573,7 +573,6 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore):
def get_destination_rooms_paginate_txn(
txn: LoggingTransaction,
) -> Tuple[List[JsonDict], int]:
-
if direction == Direction.BACKWARDS:
order = "DESC"
else:
diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py
index f6a6fd4079..c3f2b61bd5 100644
--- a/synapse/storage/databases/main/user_directory.py
+++ b/synapse/storage/databases/main/user_directory.py
@@ -98,7 +98,6 @@ class UserDirectoryBackgroundUpdateStore(StateDeltasStore):
async def _populate_user_directory_createtables(
self, progress: JsonDict, batch_size: int
) -> int:
-
# Get all the rooms that we want to process.
def _make_staging_area(txn: LoggingTransaction) -> None:
sql = (
@@ -918,11 +917,19 @@ def _parse_query_postgres(search_term: str) -> Tuple[str, str, str]:
We use this so that we can add prefix matching, which isn't something
that is supported by default.
"""
- results = _parse_words(search_term)
+ escaped_words = []
+ for word in _parse_words(search_term):
+ # Postgres tsvector and tsquery quoting rules:
+ # words potentially containing punctuation should be quoted
+ # and then existing quotes and backslashes should be doubled
+ # See: https://www.postgresql.org/docs/current/datatype-textsearch.html#DATATYPE-TSQUERY
- both = " & ".join("(%s:* | %s)" % (result, result) for result in results)
- exact = " & ".join("%s" % (result,) for result in results)
- prefix = " & ".join("%s:*" % (result,) for result in results)
+ quoted_word = word.replace("'", "''").replace("\\", "\\\\")
+ escaped_words.append(f"'{quoted_word}'")
+
+ both = " & ".join("(%s:* | %s)" % (word, word) for word in escaped_words)
+ exact = " & ".join("%s" % (word,) for word in escaped_words)
+ prefix = " & ".join("%s:*" % (word,) for word in escaped_words)
return both, exact, prefix
@@ -944,6 +951,14 @@ def _parse_words(search_term: str) -> List[str]:
if USE_ICU:
return _parse_words_with_icu(search_term)
+ return _parse_words_with_regex(search_term)
+
+
+def _parse_words_with_regex(search_term: str) -> List[str]:
+ """
+ Break down search term into words, when we don't have ICU available.
+ See: `_parse_words`
+ """
return re.findall(r"([\w\-]+)", search_term, re.UNICODE)
diff --git a/synapse/storage/databases/state/bg_updates.py b/synapse/storage/databases/state/bg_updates.py
index d743282f13..097dea5182 100644
--- a/synapse/storage/databases/state/bg_updates.py
+++ b/synapse/storage/databases/state/bg_updates.py
@@ -251,7 +251,6 @@ class StateGroupBackgroundUpdateStore(SQLBaseStore):
class StateBackgroundUpdateStore(StateGroupBackgroundUpdateStore):
-
STATE_GROUP_DEDUPLICATION_UPDATE_NAME = "state_group_state_deduplication"
STATE_GROUP_INDEX_UPDATE_NAME = "state_group_state_type_index"
STATE_GROUPS_ROOM_INDEX_UPDATE_NAME = "state_groups_room_id_idx"
diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py
index 1a7232b276..89b1faa6c8 100644
--- a/synapse/storage/databases/state/store.py
+++ b/synapse/storage/databases/state/store.py
@@ -257,14 +257,11 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore):
member_filter, non_member_filter = state_filter.get_member_split()
# Now we look them up in the member and non-member caches
- (
- non_member_state,
- incomplete_groups_nm,
- ) = self._get_state_for_groups_using_cache(
+ non_member_state, incomplete_groups_nm = self._get_state_for_groups_using_cache(
groups, self._state_group_cache, state_filter=non_member_filter
)
- (member_state, incomplete_groups_m,) = self._get_state_for_groups_using_cache(
+ member_state, incomplete_groups_m = self._get_state_for_groups_using_cache(
groups, self._state_group_members_cache, state_filter=member_filter
)
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index 6c335a9315..2a1c6fa31b 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -563,7 +563,7 @@ def _apply_module_schemas(
"""
# This is the old way for password_auth_provider modules to make changes
# to the database. This should instead be done using the module API
- for (mod, _config) in config.authproviders.password_providers:
+ for mod, _config in config.authproviders.password_providers:
if not hasattr(mod, "get_db_schema_files"):
continue
modname = ".".join((mod.__module__, mod.__name__))
@@ -591,7 +591,7 @@ def _apply_module_schema_files(
(modname,),
)
applied_deltas = {d for d, in cur}
- for (name, stream) in names_and_streams:
+ for name, stream in names_and_streams:
if name in applied_deltas:
continue
diff --git a/synapse/types/state.py b/synapse/types/state.py
index 743a4f9217..4b3071acce 100644
--- a/synapse/types/state.py
+++ b/synapse/types/state.py
@@ -120,7 +120,7 @@ class StateFilter:
def to_types(self) -> Iterable[Tuple[str, Optional[str]]]:
"""The inverse to `from_types`."""
- for (event_type, state_keys) in self.types.items():
+ for event_type, state_keys in self.types.items():
if state_keys is None:
yield event_type, None
else:
diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py
index 9387632d0d..6ffa56217e 100644
--- a/synapse/util/caches/__init__.py
+++ b/synapse/util/caches/__init__.py
@@ -98,7 +98,6 @@ class EvictionReason(Enum):
@attr.s(slots=True, auto_attribs=True)
class CacheMetric:
-
_cache: Sized
_cache_type: str
_cache_name: str
diff --git a/synapse/util/check_dependencies.py b/synapse/util/check_dependencies.py
index 3b1e205700..1c0fde4966 100644
--- a/synapse/util/check_dependencies.py
+++ b/synapse/util/check_dependencies.py
@@ -183,7 +183,7 @@ def check_requirements(extra: Optional[str] = None) -> None:
deps_unfulfilled = []
errors = []
- for (requirement, must_be_installed) in dependencies:
+ for requirement, must_be_installed in dependencies:
try:
dist: metadata.Distribution = metadata.distribution(requirement.name)
except metadata.PackageNotFoundError:
diff --git a/synapse/util/patch_inline_callbacks.py b/synapse/util/patch_inline_callbacks.py
index f97f98a057..d00d34e652 100644
--- a/synapse/util/patch_inline_callbacks.py
+++ b/synapse/util/patch_inline_callbacks.py
@@ -211,7 +211,6 @@ def _check_yield_points(
result = Failure()
if current_context() != expected_context:
-
# This happens because the context is lost sometime *after* the
# previous yield and *after* the current yield. E.g. the
# deferred we waited on didn't follow the rules, or we forgot to
diff --git a/synmark/__main__.py b/synmark/__main__.py
index 35a59e347a..19de639187 100644
--- a/synmark/__main__.py
+++ b/synmark/__main__.py
@@ -34,12 +34,10 @@ def make_test(main):
"""
def _main(loops):
-
reactor = make_reactor()
file_out = StringIO()
with redirect_stderr(file_out):
-
d = Deferred()
d.addCallback(lambda _: ensureDeferred(main(reactor, loops)))
diff --git a/synmark/suites/logging.py b/synmark/suites/logging.py
index 9419892e95..8beb077e0a 100644
--- a/synmark/suites/logging.py
+++ b/synmark/suites/logging.py
@@ -30,7 +30,6 @@ from synapse.util import Clock
class LineCounter(LineOnlyReceiver):
-
delimiter = b"\n"
def __init__(self, *args, **kwargs):
diff --git a/tests/federation/test_complexity.py b/tests/federation/test_complexity.py
index 35dd9a20df..33af8770fd 100644
--- a/tests/federation/test_complexity.py
+++ b/tests/federation/test_complexity.py
@@ -24,7 +24,6 @@ from tests.test_utils import make_awaitable
class RoomComplexityTests(unittest.FederatingHomeserverTestCase):
-
servlets = [
admin.register_servlets,
room.register_servlets,
@@ -37,7 +36,6 @@ class RoomComplexityTests(unittest.FederatingHomeserverTestCase):
return config
def test_complexity_simple(self) -> None:
-
u1 = self.register_user("u1", "pass")
u1_token = self.login("u1", "pass")
@@ -71,7 +69,6 @@ class RoomComplexityTests(unittest.FederatingHomeserverTestCase):
self.assertEqual(complexity, 1.23)
def test_join_too_large(self) -> None:
-
u1 = self.register_user("u1", "pass")
handler = self.hs.get_room_member_handler()
@@ -131,7 +128,6 @@ class RoomComplexityTests(unittest.FederatingHomeserverTestCase):
self.assertEqual(f.value.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
def test_join_too_large_once_joined(self) -> None:
-
u1 = self.register_user("u1", "pass")
u1_token = self.login("u1", "pass")
diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py
index bba6469b55..6c7738d810 100644
--- a/tests/federation/test_federation_server.py
+++ b/tests/federation/test_federation_server.py
@@ -34,7 +34,6 @@ from tests.unittest import override_config
class FederationServerTests(unittest.FederatingHomeserverTestCase):
-
servlets = [
admin.register_servlets,
room.register_servlets,
diff --git a/tests/handlers/test_admin.py b/tests/handlers/test_admin.py
index 1b97aaeed1..5569ccef8a 100644
--- a/tests/handlers/test_admin.py
+++ b/tests/handlers/test_admin.py
@@ -23,6 +23,7 @@ from synapse.api.constants import EventTypes, JoinRules
from synapse.api.room_versions import RoomVersions
from synapse.rest.client import knock, login, room
from synapse.server import HomeServer
+from synapse.types import UserID
from synapse.util import Clock
from tests import unittest
@@ -323,3 +324,31 @@ class ExfiltrateData(unittest.HomeserverTestCase):
args = writer.write_account_data.call_args_list[1][0]
self.assertEqual(args[0], "test_room")
self.assertEqual(args[1]["m.per_room"]["b"], 2)
+
+ def test_media_ids(self) -> None:
+ """Tests that media's metadata get exported."""
+
+ self.get_success(
+ self._store.store_local_media(
+ media_id="media_1",
+ media_type="image/png",
+ time_now_ms=self.clock.time_msec(),
+ upload_name=None,
+ media_length=50,
+ user_id=UserID.from_string(self.user2),
+ )
+ )
+
+ writer = Mock()
+
+ self.get_success(self.admin_handler.export_user_data(self.user2, writer))
+
+ writer.write_media_id.assert_called_once()
+
+ args = writer.write_media_id.call_args[0]
+ self.assertEqual(args[0], "media_1")
+ self.assertEqual(args[1]["media_id"], "media_1")
+ self.assertEqual(args[1]["media_length"], 50)
+ self.assertGreater(args[1]["created_ts"], 0)
+ self.assertIsNone(args[1]["upload_name"])
+ self.assertIsNone(args[1]["last_access_ts"])
diff --git a/tests/handlers/test_sso.py b/tests/handlers/test_sso.py
index 137deab138..d6f43a98fc 100644
--- a/tests/handlers/test_sso.py
+++ b/tests/handlers/test_sso.py
@@ -113,7 +113,6 @@ async def mock_get_file(
headers: Optional[RawHeaders] = None,
is_allowed_content_type: Optional[Callable[[str], bool]] = None,
) -> Tuple[int, Dict[bytes, List[bytes]], str, int]:
-
fake_response = FakeResponse(code=404)
if url == "http://my.server/me.png":
fake_response = FakeResponse(
diff --git a/tests/handlers/test_stats.py b/tests/handlers/test_stats.py
index f1a50c5bcb..d11ded6c5b 100644
--- a/tests/handlers/test_stats.py
+++ b/tests/handlers/test_stats.py
@@ -31,7 +31,6 @@ EXPT_NUM_STATE_EVTS_IN_FRESH_PRIVATE_ROOM = 6
class StatsRoomTests(unittest.HomeserverTestCase):
-
servlets = [
admin.register_servlets_for_client_rest_resource,
room.register_servlets,
diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py
index f65a68b9c2..a02c1c6227 100644
--- a/tests/handlers/test_user_directory.py
+++ b/tests/handlers/test_user_directory.py
@@ -192,6 +192,13 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase):
self.helper.join(room, self.appservice.sender, tok=self.appservice.token)
self._check_only_one_user_in_directory(user, room)
+ def test_search_term_with_colon_in_it_does_not_raise(self) -> None:
+ """
+ Regression test: Test that search terms with colons in them are acceptable.
+ """
+ u1 = self.register_user("user1", "pass")
+ self.get_success(self.handler.search_users(u1, "haha:paamayim-nekudotayim", 10))
+
def test_user_not_in_users_table(self) -> None:
"""Unclear how it happens, but on matrix.org we've seen join events
for users who aren't in the users table. Test that we don't fall over
diff --git a/tests/http/federation/test_srv_resolver.py b/tests/http/federation/test_srv_resolver.py
index 7748f56ee6..6ab13357f9 100644
--- a/tests/http/federation/test_srv_resolver.py
+++ b/tests/http/federation/test_srv_resolver.py
@@ -46,7 +46,6 @@ class SrvResolverTestCase(unittest.TestCase):
@defer.inlineCallbacks
def do_lookup() -> Generator["Deferred[object]", object, List[Server]]:
-
with LoggingContext("one") as ctx:
resolve_d = resolver.resolve_service(service_name)
result: List[Server]
diff --git a/tests/http/test_client.py b/tests/http/test_client.py
index 9cfe1ad0de..f6d6684985 100644
--- a/tests/http/test_client.py
+++ b/tests/http/test_client.py
@@ -149,7 +149,7 @@ class BlacklistingAgentTest(TestCase):
self.allowed_domain, self.allowed_ip = b"allowed.test", b"5.1.1.1"
# Configure the reactor's DNS resolver.
- for (domain, ip) in (
+ for domain, ip in (
(self.safe_domain, self.safe_ip),
(self.unsafe_domain, self.unsafe_ip),
(self.allowed_domain, self.allowed_ip),
diff --git a/tests/push/test_bulk_push_rule_evaluator.py b/tests/push/test_bulk_push_rule_evaluator.py
index 199e3d7b70..dce6899e78 100644
--- a/tests/push/test_bulk_push_rule_evaluator.py
+++ b/tests/push/test_bulk_push_rule_evaluator.py
@@ -33,7 +33,6 @@ from tests.unittest import HomeserverTestCase, override_config
class TestBulkPushRuleEvaluator(HomeserverTestCase):
-
servlets = [
admin.register_servlets_for_client_rest_resource,
room.register_servlets,
diff --git a/tests/push/test_email.py b/tests/push/test_email.py
index 7563f33fdc..0a3aca5c50 100644
--- a/tests/push/test_email.py
+++ b/tests/push/test_email.py
@@ -39,7 +39,6 @@ class _User:
class EmailPusherTests(HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets_for_client_rest_resource,
room.register_servlets,
@@ -48,7 +47,6 @@ class EmailPusherTests(HomeserverTestCase):
hijack_auth = False
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
-
config = self.default_config()
config["email"] = {
"enable_notifs": True,
diff --git a/tests/push/test_push_rule_evaluator.py b/tests/push/test_push_rule_evaluator.py
index d320a12f96..4e858fd16f 100644
--- a/tests/push/test_push_rule_evaluator.py
+++ b/tests/push/test_push_rule_evaluator.py
@@ -54,7 +54,7 @@ class FlattenDictTestCase(unittest.TestCase):
self.assertEqual({"m.foo.b\\ar": "abc"}, _flatten_dict(input))
self.assertEqual(
{"m\\.foo.b\\\\ar": "abc"},
- _flatten_dict(input, msc3783_escape_event_match_key=True),
+ _flatten_dict(input, msc3873_escape_event_match_key=True),
)
def test_non_string(self) -> None:
diff --git a/tests/replication/slave/storage/test_events.py b/tests/replication/slave/storage/test_events.py
index ddca9d696c..57c781a0c3 100644
--- a/tests/replication/slave/storage/test_events.py
+++ b/tests/replication/slave/storage/test_events.py
@@ -64,7 +64,6 @@ def patch__eq__(cls: object) -> Callable[[], None]:
class EventsWorkerStoreTestCase(BaseSlavedStoreTestCase):
-
STORE_TYPE = EventsWorkerStore
def setUp(self) -> None:
diff --git a/tests/rest/admin/test_device.py b/tests/rest/admin/test_device.py
index 03f2112b07..aaa488bced 100644
--- a/tests/rest/admin/test_device.py
+++ b/tests/rest/admin/test_device.py
@@ -28,7 +28,6 @@ from tests import unittest
class DeviceRestTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets,
login.register_servlets,
@@ -291,7 +290,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase):
class DevicesRestTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets,
login.register_servlets,
@@ -415,7 +413,6 @@ class DevicesRestTestCase(unittest.HomeserverTestCase):
class DeleteDevicesRestTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets,
login.register_servlets,
diff --git a/tests/rest/admin/test_media.py b/tests/rest/admin/test_media.py
index db77a45ae3..f41319a5b6 100644
--- a/tests/rest/admin/test_media.py
+++ b/tests/rest/admin/test_media.py
@@ -34,7 +34,6 @@ INVALID_TIMESTAMP_IN_S = 1893456000 # 2030-01-01 in seconds
class DeleteMediaByIDTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets,
synapse.rest.admin.register_servlets_for_media_repo,
@@ -196,7 +195,6 @@ class DeleteMediaByIDTestCase(unittest.HomeserverTestCase):
class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets,
synapse.rest.admin.register_servlets_for_media_repo,
@@ -594,7 +592,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase):
class QuarantineMediaByIDTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets,
synapse.rest.admin.register_servlets_for_media_repo,
@@ -724,7 +721,6 @@ class QuarantineMediaByIDTestCase(unittest.HomeserverTestCase):
class ProtectMediaByIDTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets,
synapse.rest.admin.register_servlets_for_media_repo,
@@ -821,7 +817,6 @@ class ProtectMediaByIDTestCase(unittest.HomeserverTestCase):
class PurgeMediaCacheTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets,
synapse.rest.admin.register_servlets_for_media_repo,
diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py
index 453a6e979c..9dbb778679 100644
--- a/tests/rest/admin/test_room.py
+++ b/tests/rest/admin/test_room.py
@@ -1990,7 +1990,6 @@ class RoomMessagesTestCase(unittest.HomeserverTestCase):
class JoinAliasRoomTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets,
room.register_servlets,
diff --git a/tests/rest/admin/test_server_notice.py b/tests/rest/admin/test_server_notice.py
index f71ff46d87..28b999573e 100644
--- a/tests/rest/admin/test_server_notice.py
+++ b/tests/rest/admin/test_server_notice.py
@@ -28,7 +28,6 @@ from tests.unittest import override_config
class ServerNoticeTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets,
login.register_servlets,
diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py
index e2ee1a1766..2b05dffc7d 100644
--- a/tests/rest/client/test_account.py
+++ b/tests/rest/client/test_account.py
@@ -40,7 +40,6 @@ from tests.unittest import override_config
class PasswordResetTestCase(unittest.HomeserverTestCase):
-
servlets = [
account.register_servlets,
synapse.rest.admin.register_servlets_for_client_rest_resource,
@@ -408,7 +407,6 @@ class PasswordResetTestCase(unittest.HomeserverTestCase):
class DeactivateTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets_for_client_rest_resource,
login.register_servlets,
@@ -492,7 +490,6 @@ class DeactivateTestCase(unittest.HomeserverTestCase):
class WhoamiTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets_for_client_rest_resource,
login.register_servlets,
@@ -567,7 +564,6 @@ class WhoamiTestCase(unittest.HomeserverTestCase):
class ThreepidEmailRestTestCase(unittest.HomeserverTestCase):
-
servlets = [
account.register_servlets,
login.register_servlets,
diff --git a/tests/rest/client/test_auth.py b/tests/rest/client/test_auth.py
index a144610078..0d8fe77b88 100644
--- a/tests/rest/client/test_auth.py
+++ b/tests/rest/client/test_auth.py
@@ -52,7 +52,6 @@ class DummyRecaptchaChecker(UserInteractiveAuthChecker):
class FallbackAuthTests(unittest.HomeserverTestCase):
-
servlets = [
auth.register_servlets,
register.register_servlets,
@@ -60,7 +59,6 @@ class FallbackAuthTests(unittest.HomeserverTestCase):
hijack_auth = False
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
-
config = self.default_config()
config["enable_registration_captcha"] = True
diff --git a/tests/rest/client/test_capabilities.py b/tests/rest/client/test_capabilities.py
index d1751e1557..c16e8d43f4 100644
--- a/tests/rest/client/test_capabilities.py
+++ b/tests/rest/client/test_capabilities.py
@@ -26,7 +26,6 @@ from tests.unittest import override_config
class CapabilitiesTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets_for_client_rest_resource,
capabilities.register_servlets,
diff --git a/tests/rest/client/test_consent.py b/tests/rest/client/test_consent.py
index b1ca81a911..bb845179d3 100644
--- a/tests/rest/client/test_consent.py
+++ b/tests/rest/client/test_consent.py
@@ -38,7 +38,6 @@ class ConsentResourceTestCase(unittest.HomeserverTestCase):
hijack_auth = False
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
-
config = self.default_config()
config["form_secret"] = "123abc"
diff --git a/tests/rest/client/test_directory.py b/tests/rest/client/test_directory.py
index 7a88aa2cda..6490e883bf 100644
--- a/tests/rest/client/test_directory.py
+++ b/tests/rest/client/test_directory.py
@@ -28,7 +28,6 @@ from tests.unittest import override_config
class DirectoryTestCase(unittest.HomeserverTestCase):
-
servlets = [
admin.register_servlets_for_client_rest_resource,
directory.register_servlets,
diff --git a/tests/rest/client/test_ephemeral_message.py b/tests/rest/client/test_ephemeral_message.py
index 9fa1f82dfe..f31ebc8021 100644
--- a/tests/rest/client/test_ephemeral_message.py
+++ b/tests/rest/client/test_ephemeral_message.py
@@ -26,7 +26,6 @@ from tests import unittest
class EphemeralMessageTestCase(unittest.HomeserverTestCase):
-
user_id = "@user:test"
servlets = [
diff --git a/tests/rest/client/test_events.py b/tests/rest/client/test_events.py
index a9b7db9db2..54df2a252c 100644
--- a/tests/rest/client/test_events.py
+++ b/tests/rest/client/test_events.py
@@ -38,7 +38,6 @@ class EventStreamPermissionsTestCase(unittest.HomeserverTestCase):
]
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
-
config = self.default_config()
config["enable_registration_captcha"] = False
config["enable_registration"] = True
@@ -51,7 +50,6 @@ class EventStreamPermissionsTestCase(unittest.HomeserverTestCase):
return hs
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
-
# register an account
self.user_id = self.register_user("sid1", "pass")
self.token = self.login(self.user_id, "pass")
@@ -142,7 +140,6 @@ class GetEventsTestCase(unittest.HomeserverTestCase):
]
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
-
# register an account
self.user_id = self.register_user("sid1", "pass")
self.token = self.login(self.user_id, "pass")
diff --git a/tests/rest/client/test_filter.py b/tests/rest/client/test_filter.py
index 830762fd53..91678abf13 100644
--- a/tests/rest/client/test_filter.py
+++ b/tests/rest/client/test_filter.py
@@ -25,7 +25,6 @@ PATH_PREFIX = "/_matrix/client/v2_alpha"
class FilterTestCase(unittest.HomeserverTestCase):
-
user_id = "@apple:test"
hijack_auth = True
EXAMPLE_FILTER = {"room": {"timeline": {"types": ["m.room.message"]}}}
diff --git a/tests/rest/client/test_login.py b/tests/rest/client/test_login.py
index ff5baa9f0a..62acf4f44e 100644
--- a/tests/rest/client/test_login.py
+++ b/tests/rest/client/test_login.py
@@ -89,7 +89,6 @@ ADDITIONAL_LOGIN_FLOWS = [
class LoginRestServletTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets_for_client_rest_resource,
login.register_servlets,
@@ -737,7 +736,6 @@ class MultiSSOTestCase(unittest.HomeserverTestCase):
class CASTestCase(unittest.HomeserverTestCase):
-
servlets = [
login.register_servlets,
]
diff --git a/tests/rest/client/test_login_token_request.py b/tests/rest/client/test_login_token_request.py
index 6aedc1a11c..b8187db982 100644
--- a/tests/rest/client/test_login_token_request.py
+++ b/tests/rest/client/test_login_token_request.py
@@ -26,7 +26,6 @@ endpoint = "/_matrix/client/unstable/org.matrix.msc3882/login/token"
class LoginTokenRequestServletTestCase(unittest.HomeserverTestCase):
-
servlets = [
login.register_servlets,
admin.register_servlets,
diff --git a/tests/rest/client/test_presence.py b/tests/rest/client/test_presence.py
index 67e16880e6..dcbb125a3b 100644
--- a/tests/rest/client/test_presence.py
+++ b/tests/rest/client/test_presence.py
@@ -35,7 +35,6 @@ class PresenceTestCase(unittest.HomeserverTestCase):
servlets = [presence.register_servlets]
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
-
self.presence_handler = Mock(spec=PresenceHandler)
self.presence_handler.set_state.return_value = make_awaitable(None)
diff --git a/tests/rest/client/test_profile.py b/tests/rest/client/test_profile.py
index 8de5a342ae..27c93ad761 100644
--- a/tests/rest/client/test_profile.py
+++ b/tests/rest/client/test_profile.py
@@ -30,7 +30,6 @@ from tests import unittest
class ProfileTestCase(unittest.HomeserverTestCase):
-
servlets = [
admin.register_servlets_for_client_rest_resource,
login.register_servlets,
@@ -324,7 +323,6 @@ class ProfileTestCase(unittest.HomeserverTestCase):
class ProfilesRestrictedTestCase(unittest.HomeserverTestCase):
-
servlets = [
admin.register_servlets_for_client_rest_resource,
login.register_servlets,
@@ -404,7 +402,6 @@ class ProfilesRestrictedTestCase(unittest.HomeserverTestCase):
class OwnProfileUnrestrictedTestCase(unittest.HomeserverTestCase):
-
servlets = [
admin.register_servlets_for_client_rest_resource,
login.register_servlets,
diff --git a/tests/rest/client/test_register.py b/tests/rest/client/test_register.py
index 4c561f9525..b228dba861 100644
--- a/tests/rest/client/test_register.py
+++ b/tests/rest/client/test_register.py
@@ -40,7 +40,6 @@ from tests.unittest import override_config
class RegisterRestServletTestCase(unittest.HomeserverTestCase):
-
servlets = [
login.register_servlets,
register.register_servlets,
@@ -797,7 +796,6 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase):
class AccountValidityTestCase(unittest.HomeserverTestCase):
-
servlets = [
register.register_servlets,
synapse.rest.admin.register_servlets_for_client_rest_resource,
@@ -913,7 +911,6 @@ class AccountValidityTestCase(unittest.HomeserverTestCase):
class AccountValidityRenewalByEmailTestCase(unittest.HomeserverTestCase):
-
servlets = [
register.register_servlets,
synapse.rest.admin.register_servlets_for_client_rest_resource,
@@ -1132,7 +1129,6 @@ class AccountValidityRenewalByEmailTestCase(unittest.HomeserverTestCase):
class AccountValidityBackgroundJobTestCase(unittest.HomeserverTestCase):
-
servlets = [synapse.rest.admin.register_servlets_for_client_rest_resource]
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
diff --git a/tests/rest/client/test_rendezvous.py b/tests/rest/client/test_rendezvous.py
index c0eb5d01a6..8dbd64be55 100644
--- a/tests/rest/client/test_rendezvous.py
+++ b/tests/rest/client/test_rendezvous.py
@@ -25,7 +25,6 @@ endpoint = "/_matrix/client/unstable/org.matrix.msc3886/rendezvous"
class RendezvousServletTestCase(unittest.HomeserverTestCase):
-
servlets = [
rendezvous.register_servlets,
]
diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py
index cfad182b2f..4dd763096d 100644
--- a/tests/rest/client/test_rooms.py
+++ b/tests/rest/client/test_rooms.py
@@ -65,7 +65,6 @@ class RoomBase(unittest.HomeserverTestCase):
servlets = [room.register_servlets, room.register_deprecated_servlets]
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
-
self.hs = self.setup_test_homeserver(
"red",
federation_http_client=None,
@@ -92,7 +91,6 @@ class RoomPermissionsTestCase(RoomBase):
rmcreator_id = "@notme:red"
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
-
self.helper.auth_user_id = self.rmcreator_id
# create some rooms under the name rmcreator_id
self.uncreated_rmid = "!aa:test"
@@ -1127,7 +1125,6 @@ class RoomInviteRatelimitTestCase(RoomBase):
class RoomJoinTestCase(RoomBase):
-
servlets = [
admin.register_servlets,
login.register_servlets,
@@ -2102,7 +2099,6 @@ class RoomSearchTestCase(unittest.HomeserverTestCase):
hijack_auth = False
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
-
# Register the user who does the searching
self.user_id2 = self.register_user("user", "pass")
self.access_token = self.login("user", "pass")
@@ -2195,7 +2191,6 @@ class RoomSearchTestCase(unittest.HomeserverTestCase):
class PublicRoomsRestrictedTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets_for_client_rest_resource,
room.register_servlets,
@@ -2203,7 +2198,6 @@ class PublicRoomsRestrictedTestCase(unittest.HomeserverTestCase):
]
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
-
self.url = b"/_matrix/client/r0/publicRooms"
config = self.default_config()
@@ -2225,7 +2219,6 @@ class PublicRoomsRestrictedTestCase(unittest.HomeserverTestCase):
class PublicRoomsRoomTypeFilterTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets_for_client_rest_resource,
room.register_servlets,
@@ -2233,7 +2226,6 @@ class PublicRoomsRoomTypeFilterTestCase(unittest.HomeserverTestCase):
]
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
-
config = self.default_config()
config["allow_public_rooms_without_auth"] = True
self.hs = self.setup_test_homeserver(config=config)
@@ -2414,7 +2406,6 @@ class PublicRoomsTestRemoteSearchFallbackTestCase(unittest.HomeserverTestCase):
class PerRoomProfilesForbiddenTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets_for_client_rest_resource,
room.register_servlets,
@@ -2983,7 +2974,6 @@ class RelationsTestCase(PaginationTestCase):
class ContextTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets_for_client_rest_resource,
room.register_servlets,
@@ -3359,7 +3349,6 @@ class RoomCanonicalAliasTestCase(unittest.HomeserverTestCase):
class ThreepidInviteTestCase(unittest.HomeserverTestCase):
-
servlets = [
admin.register_servlets,
login.register_servlets,
@@ -3438,7 +3427,8 @@ class ThreepidInviteTestCase(unittest.HomeserverTestCase):
"""
Test allowing/blocking threepid invites with a spam-check module.
- In this test, we use the more recent API in which callbacks return a `Union[Codes, Literal["NOT_SPAM"]]`."""
+ In this test, we use the more recent API in which callbacks return a `Union[Codes, Literal["NOT_SPAM"]]`.
+ """
# Mock a few functions to prevent the test from failing due to failing to talk to
# a remote IS. We keep the mock for make_and_store_3pid_invite around so we
# can check its call_count later on during the test.
diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py
index b9047194dd..9c876c7a32 100644
--- a/tests/rest/client/test_sync.py
+++ b/tests/rest/client/test_sync.py
@@ -41,7 +41,6 @@ from tests.server import TimedOutException
class FilterTestCase(unittest.HomeserverTestCase):
-
user_id = "@apple:test"
servlets = [
synapse.rest.admin.register_servlets_for_client_rest_resource,
@@ -191,7 +190,6 @@ class SyncFilterTestCase(unittest.HomeserverTestCase):
class SyncTypingTests(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets_for_client_rest_resource,
room.register_servlets,
@@ -892,7 +890,6 @@ class DeviceListSyncTestCase(unittest.HomeserverTestCase):
class ExcludeRoomTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets,
login.register_servlets,
diff --git a/tests/rest/client/test_third_party_rules.py b/tests/rest/client/test_third_party_rules.py
index 5fa3440691..c0f93f898a 100644
--- a/tests/rest/client/test_third_party_rules.py
+++ b/tests/rest/client/test_third_party_rules.py
@@ -137,6 +137,7 @@ class ThirdPartyRulesTestCase(unittest.FederatingHomeserverTestCase):
"""Tests that a forbidden event is forbidden from being sent, but an allowed one
can be sent.
"""
+
# patch the rules module with a Mock which will return False for some event
# types
async def check(
@@ -243,6 +244,7 @@ class ThirdPartyRulesTestCase(unittest.FederatingHomeserverTestCase):
def test_modify_event(self) -> None:
"""The module can return a modified version of the event"""
+
# first patch the event checker so that it will modify the event
async def check(
ev: EventBase, state: StateMap[EventBase]
@@ -275,6 +277,7 @@ class ThirdPartyRulesTestCase(unittest.FederatingHomeserverTestCase):
def test_message_edit(self) -> None:
"""Ensure that the module doesn't cause issues with edited messages."""
+
# first patch the event checker so that it will modify the event
async def check(
ev: EventBase, state: StateMap[EventBase]
diff --git a/tests/rest/media/test_media_retention.py b/tests/rest/media/test_media_retention.py
index 23f227aed6..b59d9dfd4d 100644
--- a/tests/rest/media/test_media_retention.py
+++ b/tests/rest/media/test_media_retention.py
@@ -31,7 +31,6 @@ from tests.utils import MockClock
class MediaRetentionTestCase(unittest.HomeserverTestCase):
-
ONE_DAY_IN_MS = 24 * 60 * 60 * 1000
THIRTY_DAYS_IN_MS = 30 * ONE_DAY_IN_MS
diff --git a/tests/rest/media/v1/test_media_storage.py b/tests/rest/media/v1/test_media_storage.py
index 17a3b06a8e..8ed27179c4 100644
--- a/tests/rest/media/v1/test_media_storage.py
+++ b/tests/rest/media/v1/test_media_storage.py
@@ -52,7 +52,6 @@ from tests.utils import default_config
class MediaStorageTests(unittest.HomeserverTestCase):
-
needs_threadpool = True
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
@@ -207,7 +206,6 @@ class MediaRepoTests(unittest.HomeserverTestCase):
user_id = "@test:user"
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
-
self.fetches: List[
Tuple[
"Deferred[Tuple[bytes, Tuple[int, Dict[bytes, List[bytes]]]]]",
@@ -268,7 +266,6 @@ class MediaRepoTests(unittest.HomeserverTestCase):
return hs
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
-
media_resource = hs.get_media_repository_resource()
self.download_resource = media_resource.children[b"download"]
self.thumbnail_resource = media_resource.children[b"thumbnail"]
diff --git a/tests/rest/media/v1/test_url_preview.py b/tests/rest/media/v1/test_url_preview.py
index 2c321f8d04..2acfccec61 100644
--- a/tests/rest/media/v1/test_url_preview.py
+++ b/tests/rest/media/v1/test_url_preview.py
@@ -58,7 +58,6 @@ class URLPreviewTests(unittest.HomeserverTestCase):
)
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
-
config = self.default_config()
config["url_preview_enabled"] = True
config["max_spider_size"] = 9999999
@@ -118,7 +117,6 @@ class URLPreviewTests(unittest.HomeserverTestCase):
return hs
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
-
self.media_repo = hs.get_media_repository_resource()
self.preview_url = self.media_repo.children[b"preview_url"]
@@ -133,7 +131,6 @@ class URLPreviewTests(unittest.HomeserverTestCase):
addressTypes: Optional[Sequence[Type[IAddress]]] = None,
transportSemantics: str = "TCP",
) -> IResolutionReceiver:
-
resolution = HostResolution(hostName)
resolutionReceiver.resolutionBegan(resolution)
if hostName not in self.lookups:
@@ -660,7 +657,7 @@ class URLPreviewTests(unittest.HomeserverTestCase):
"""If the preview image doesn't exist, ensure some data is returned."""
self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
- end_content = (
+ result = (
b"""<html><body><img src="http://cdn.matrix.org/foo.jpg"></body></html>"""
)
@@ -681,8 +678,8 @@ class URLPreviewTests(unittest.HomeserverTestCase):
b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
b'Content-Type: text/html; charset="utf8"\r\n\r\n'
)
- % (len(end_content),)
- + end_content
+ % (len(result),)
+ + result
)
self.pump()
@@ -691,6 +688,44 @@ class URLPreviewTests(unittest.HomeserverTestCase):
# The image should not be in the result.
self.assertNotIn("og:image", channel.json_body)
+ def test_oembed_failure(self) -> None:
+ """If the autodiscovered oEmbed URL fails, ensure some data is returned."""
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
+
+ result = b"""
+ <title>oEmbed Autodiscovery Fail</title>
+ <link rel="alternate" type="application/json+oembed"
+ href="http://example.com/oembed?url=http%3A%2F%2Fmatrix.org&format=json"
+ title="matrixdotorg" />
+ """
+
+ channel = self.make_request(
+ "GET",
+ "preview_url?url=http://matrix.org",
+ shorthand=False,
+ await_result=False,
+ )
+ self.pump()
+
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
+ server = AccumulatingProtocol()
+ server.makeConnection(FakeTransport(client, self.reactor))
+ client.makeConnection(FakeTransport(server, self.reactor))
+ client.dataReceived(
+ (
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
+ b'Content-Type: text/html; charset="utf8"\r\n\r\n'
+ )
+ % (len(result),)
+ + result
+ )
+
+ self.pump()
+ self.assertEqual(channel.code, 200)
+
+ # The image should not be in the result.
+ self.assertEqual(channel.json_body["og:title"], "oEmbed Autodiscovery Fail")
+
def test_data_url(self) -> None:
"""
Requesting to preview a data URL is not supported.
diff --git a/tests/server_notices/test_consent.py b/tests/server_notices/test_consent.py
index 6540ed53f1..3fdf5a6d52 100644
--- a/tests/server_notices/test_consent.py
+++ b/tests/server_notices/test_consent.py
@@ -25,7 +25,6 @@ from tests import unittest
class ConsentNoticesTests(unittest.HomeserverTestCase):
-
servlets = [
sync.register_servlets,
synapse.rest.admin.register_servlets_for_client_rest_resource,
@@ -34,7 +33,6 @@ class ConsentNoticesTests(unittest.HomeserverTestCase):
]
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
-
tmpdir = self.mktemp()
os.mkdir(tmpdir)
self.consent_notice_message = "consent %(consent_uri)s"
diff --git a/tests/storage/databases/main/test_deviceinbox.py b/tests/storage/databases/main/test_deviceinbox.py
index 373707b275..b6d5c474b0 100644
--- a/tests/storage/databases/main/test_deviceinbox.py
+++ b/tests/storage/databases/main/test_deviceinbox.py
@@ -23,7 +23,6 @@ from tests.unittest import HomeserverTestCase
class DeviceInboxBackgroundUpdateStoreTestCase(HomeserverTestCase):
-
servlets = [
admin.register_servlets,
devices.register_servlets,
diff --git a/tests/storage/databases/main/test_receipts.py b/tests/storage/databases/main/test_receipts.py
index ac77aec003..71db47405e 100644
--- a/tests/storage/databases/main/test_receipts.py
+++ b/tests/storage/databases/main/test_receipts.py
@@ -26,7 +26,6 @@ from tests.unittest import HomeserverTestCase
class ReceiptsBackgroundUpdateStoreTestCase(HomeserverTestCase):
-
servlets = [
admin.register_servlets,
room.register_servlets,
@@ -62,6 +61,7 @@ class ReceiptsBackgroundUpdateStoreTestCase(HomeserverTestCase):
keys and expected receipt key-values after duplicate receipts have been
removed.
"""
+
# First, undo the background update.
def drop_receipts_unique_index(txn: LoggingTransaction) -> None:
txn.execute(f"DROP INDEX IF EXISTS {index_name}")
diff --git a/tests/storage/databases/main/test_room.py b/tests/storage/databases/main/test_room.py
index 3108ca3444..dbd8f3a85e 100644
--- a/tests/storage/databases/main/test_room.py
+++ b/tests/storage/databases/main/test_room.py
@@ -27,7 +27,6 @@ from tests.unittest import HomeserverTestCase
class RoomBackgroundUpdateStoreTestCase(HomeserverTestCase):
-
servlets = [
admin.register_servlets,
room.register_servlets,
diff --git a/tests/storage/test_cleanup_extrems.py b/tests/storage/test_cleanup_extrems.py
index d570684c99..7de109966d 100644
--- a/tests/storage/test_cleanup_extrems.py
+++ b/tests/storage/test_cleanup_extrems.py
@@ -43,8 +43,9 @@ class CleanupExtremBackgroundUpdateStoreTestCase(HomeserverTestCase):
# Create a test user and room
self.user = UserID("alice", "test")
self.requester = create_requester(self.user)
- info, _ = self.get_success(self.room_creator.create_room(self.requester, {}))
- self.room_id = info["room_id"]
+ self.room_id, _, _ = self.get_success(
+ self.room_creator.create_room(self.requester, {})
+ )
def run_background_update(self) -> None:
"""Re run the background update to clean up the extremities."""
@@ -275,10 +276,9 @@ class CleanupExtremDummyEventsTestCase(HomeserverTestCase):
self.user = UserID.from_string(self.register_user("user1", "password"))
self.token1 = self.login("user1", "password")
self.requester = create_requester(self.user)
- info, _ = self.get_success(
+ self.room_id, _, _ = self.get_success(
self.room_creator.create_room(self.requester, {"visibility": "public"})
)
- self.room_id = info["room_id"]
self.event_creator = homeserver.get_event_creation_handler()
homeserver.config.consent.user_consent_version = self.CONSENT_VERSION
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index 7f7f4ef892..cd0079871c 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -656,7 +656,6 @@ class ClientIpStoreTestCase(unittest.HomeserverTestCase):
class ClientIpAuthTestCase(unittest.HomeserverTestCase):
-
servlets = [
synapse.rest.admin.register_servlets,
login.register_servlets,
diff --git a/tests/storage/test_event_chain.py b/tests/storage/test_event_chain.py
index a10e5fa8b1..73d11e7786 100644
--- a/tests/storage/test_event_chain.py
+++ b/tests/storage/test_event_chain.py
@@ -417,7 +417,6 @@ class EventChainStoreTestCase(HomeserverTestCase):
def fetch_chains(
self, events: List[EventBase]
) -> Tuple[Dict[str, Tuple[int, int]], _LinkMap]:
-
# Fetch the map from event ID -> (chain ID, sequence number)
rows = self.get_success(
self.store.db_pool.simple_select_many_batch(
@@ -492,7 +491,6 @@ class LinkMapTestCase(unittest.TestCase):
class EventChainBackgroundUpdateTestCase(HomeserverTestCase):
-
servlets = [
admin.register_servlets,
room.register_servlets,
diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py
index 8fc7936ab0..3e1984c15c 100644
--- a/tests/storage/test_event_federation.py
+++ b/tests/storage/test_event_federation.py
@@ -672,7 +672,7 @@ class EventFederationWorkerStoreTestCase(tests.unittest.HomeserverTestCase):
complete_event_dict_map: Dict[str, JsonDict] = {}
stream_ordering = 0
- for (event_id, prev_event_ids) in event_graph.items():
+ for event_id, prev_event_ids in event_graph.items():
depth = depth_map[event_id]
complete_event_dict_map[event_id] = {
diff --git a/tests/storage/test_event_metrics.py b/tests/storage/test_event_metrics.py
index a91411168c..6897addbd3 100644
--- a/tests/storage/test_event_metrics.py
+++ b/tests/storage/test_event_metrics.py
@@ -33,8 +33,7 @@ class ExtremStatisticsTestCase(HomeserverTestCase):
events = [(3, 2), (6, 2), (4, 6)]
for event_count, extrems in events:
- info, _ = self.get_success(room_creator.create_room(requester, {}))
- room_id = info["room_id"]
+ room_id, _, _ = self.get_success(room_creator.create_room(requester, {}))
last_event = None
diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py
index 76c06a9d1e..aa19c3bd30 100644
--- a/tests/storage/test_event_push_actions.py
+++ b/tests/storage/test_event_push_actions.py
@@ -774,7 +774,7 @@ class EventPushActionsStoreTestCase(HomeserverTestCase):
self.assertEqual(r, 3)
# add a bunch of dummy events to the events table
- for (stream_ordering, ts) in (
+ for stream_ordering, ts in (
(3, 110),
(4, 120),
(5, 120),
diff --git a/tests/storage/test_purge.py b/tests/storage/test_purge.py
index d8f42c5d05..857e2caf2e 100644
--- a/tests/storage/test_purge.py
+++ b/tests/storage/test_purge.py
@@ -23,7 +23,6 @@ from tests.unittest import HomeserverTestCase
class PurgeTests(HomeserverTestCase):
-
user_id = "@red:server"
servlets = [room.register_servlets]
diff --git a/tests/storage/test_receipts.py b/tests/storage/test_receipts.py
index 12c17f1073..1b52eef23f 100644
--- a/tests/storage/test_receipts.py
+++ b/tests/storage/test_receipts.py
@@ -50,12 +50,14 @@ class ReceiptTestCase(HomeserverTestCase):
self.otherRequester = create_requester(self.otherUser)
# Create a test room
- info, _ = self.get_success(self.room_creator.create_room(self.ourRequester, {}))
- self.room_id1 = info["room_id"]
+ self.room_id1, _, _ = self.get_success(
+ self.room_creator.create_room(self.ourRequester, {})
+ )
# Create a second test room
- info, _ = self.get_success(self.room_creator.create_room(self.ourRequester, {}))
- self.room_id2 = info["room_id"]
+ self.room_id2, _, _ = self.get_success(
+ self.room_creator.create_room(self.ourRequester, {})
+ )
# Join the second user to the first room
memberEvent, memberEventContext = self.get_success(
diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py
index 8794401823..f4c4661aaf 100644
--- a/tests/storage/test_roommember.py
+++ b/tests/storage/test_roommember.py
@@ -27,7 +27,6 @@ from tests.test_utils import event_injection
class RoomMemberStoreTestCase(unittest.HomeserverTestCase):
-
servlets = [
login.register_servlets,
register_servlets_for_client_rest_resource,
@@ -35,7 +34,6 @@ class RoomMemberStoreTestCase(unittest.HomeserverTestCase):
]
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: TestHomeServer) -> None: # type: ignore[override]
-
# We can't test the RoomMemberStore on its own without the other event
# storage logic
self.store = hs.get_datastores().main
@@ -48,7 +46,6 @@ class RoomMemberStoreTestCase(unittest.HomeserverTestCase):
self.u_charlie = UserID.from_string("@charlie:elsewhere")
def test_one_member(self) -> None:
-
# Alice creates the room, and is automatically joined
self.room = self.helper.create_room_as(self.u_alice, tok=self.t_alice)
diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index f730b888f7..e82c03f597 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -242,7 +242,7 @@ class StateStoreTestCase(HomeserverTestCase):
# test _get_state_for_group_using_cache correctly filters out members
# with types=[]
- (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
+ state_dict, is_all = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_cache,
group,
state_filter=StateFilter(
@@ -259,7 +259,7 @@ class StateStoreTestCase(HomeserverTestCase):
state_dict,
)
- (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
+ state_dict, is_all = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_members_cache,
group,
state_filter=StateFilter(
@@ -272,7 +272,7 @@ class StateStoreTestCase(HomeserverTestCase):
# test _get_state_for_group_using_cache correctly filters in members
# with wildcard types
- (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
+ state_dict, is_all = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_cache,
group,
state_filter=StateFilter(
@@ -289,7 +289,7 @@ class StateStoreTestCase(HomeserverTestCase):
state_dict,
)
- (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
+ state_dict, is_all = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_members_cache,
group,
state_filter=StateFilter(
@@ -309,7 +309,7 @@ class StateStoreTestCase(HomeserverTestCase):
# test _get_state_for_group_using_cache correctly filters in members
# with specific types
- (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
+ state_dict, is_all = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_cache,
group,
state_filter=StateFilter(
@@ -327,7 +327,7 @@ class StateStoreTestCase(HomeserverTestCase):
state_dict,
)
- (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
+ state_dict, is_all = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_members_cache,
group,
state_filter=StateFilter(
@@ -341,7 +341,7 @@ class StateStoreTestCase(HomeserverTestCase):
# test _get_state_for_group_using_cache correctly filters in members
# with specific types
- (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
+ state_dict, is_all = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_members_cache,
group,
state_filter=StateFilter(
@@ -392,7 +392,7 @@ class StateStoreTestCase(HomeserverTestCase):
# test _get_state_for_group_using_cache correctly filters out members
# with types=[]
room_id = self.room.to_string()
- (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
+ state_dict, is_all = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_cache,
group,
state_filter=StateFilter(
@@ -404,7 +404,7 @@ class StateStoreTestCase(HomeserverTestCase):
self.assertDictEqual({}, state_dict)
room_id = self.room.to_string()
- (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
+ state_dict, is_all = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_members_cache,
group,
state_filter=StateFilter(
@@ -417,7 +417,7 @@ class StateStoreTestCase(HomeserverTestCase):
# test _get_state_for_group_using_cache correctly filters in members
# wildcard types
- (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
+ state_dict, is_all = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_cache,
group,
state_filter=StateFilter(
@@ -428,7 +428,7 @@ class StateStoreTestCase(HomeserverTestCase):
self.assertEqual(is_all, False)
self.assertDictEqual({}, state_dict)
- (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
+ state_dict, is_all = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_members_cache,
group,
state_filter=StateFilter(
@@ -447,7 +447,7 @@ class StateStoreTestCase(HomeserverTestCase):
# test _get_state_for_group_using_cache correctly filters in members
# with specific types
- (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
+ state_dict, is_all = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_cache,
group,
state_filter=StateFilter(
@@ -459,7 +459,7 @@ class StateStoreTestCase(HomeserverTestCase):
self.assertEqual(is_all, False)
self.assertDictEqual({}, state_dict)
- (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
+ state_dict, is_all = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_members_cache,
group,
state_filter=StateFilter(
@@ -473,7 +473,7 @@ class StateStoreTestCase(HomeserverTestCase):
# test _get_state_for_group_using_cache correctly filters in members
# with specific types
- (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
+ state_dict, is_all = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_cache,
group,
state_filter=StateFilter(
@@ -485,7 +485,7 @@ class StateStoreTestCase(HomeserverTestCase):
self.assertEqual(is_all, False)
self.assertDictEqual({}, state_dict)
- (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
+ state_dict, is_all = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_members_cache,
group,
state_filter=StateFilter(
diff --git a/tests/storage/test_user_directory.py b/tests/storage/test_user_directory.py
index f1ca523d23..2d169684cf 100644
--- a/tests/storage/test_user_directory.py
+++ b/tests/storage/test_user_directory.py
@@ -25,6 +25,11 @@ from synapse.rest.client import login, register, room
from synapse.server import HomeServer
from synapse.storage import DataStore
from synapse.storage.background_updates import _BackgroundUpdateHandler
+from synapse.storage.databases.main import user_directory
+from synapse.storage.databases.main.user_directory import (
+ _parse_words_with_icu,
+ _parse_words_with_regex,
+)
from synapse.storage.roommember import ProfileInfo
from synapse.util import Clock
@@ -42,7 +47,7 @@ ALICE = "@alice:a"
BOB = "@bob:b"
BOBBY = "@bobby:a"
# The localpart isn't 'Bela' on purpose so we can test looking up display names.
-BELA = "@somenickname:a"
+BELA = "@somenickname:example.org"
class GetUserDirectoryTables:
@@ -423,6 +428,8 @@ class UserDirectoryInitialPopulationTestcase(HomeserverTestCase):
class UserDirectoryStoreTestCase(HomeserverTestCase):
+ use_icu = False
+
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
self.store = hs.get_datastores().main
@@ -434,6 +441,12 @@ class UserDirectoryStoreTestCase(HomeserverTestCase):
self.get_success(self.store.update_profile_in_user_dir(BELA, "Bela", None))
self.get_success(self.store.add_users_in_public_rooms("!room:id", (ALICE, BOB)))
+ self._restore_use_icu = user_directory.USE_ICU
+ user_directory.USE_ICU = self.use_icu
+
+ def tearDown(self) -> None:
+ user_directory.USE_ICU = self._restore_use_icu
+
def test_search_user_dir(self) -> None:
# normally when alice searches the directory she should just find
# bob because bobby doesn't share a room with her.
@@ -478,6 +491,26 @@ class UserDirectoryStoreTestCase(HomeserverTestCase):
{"user_id": BELA, "display_name": "Bela", "avatar_url": None},
)
+ @override_config({"user_directory": {"search_all_users": True}})
+ def test_search_user_dir_start_of_user_id(self) -> None:
+ """Tests that a user can look up another user by searching for the start
+ of their user ID.
+ """
+ r = self.get_success(self.store.search_user_dir(ALICE, "somenickname:exa", 10))
+ self.assertFalse(r["limited"])
+ self.assertEqual(1, len(r["results"]))
+ self.assertDictEqual(
+ r["results"][0],
+ {"user_id": BELA, "display_name": "Bela", "avatar_url": None},
+ )
+
+
+class UserDirectoryStoreTestCaseWithIcu(UserDirectoryStoreTestCase):
+ use_icu = True
+
+ if not icu:
+ skip = "Requires PyICU"
+
class UserDirectoryICUTestCase(HomeserverTestCase):
if not icu:
@@ -513,3 +546,31 @@ class UserDirectoryICUTestCase(HomeserverTestCase):
r["results"][0],
{"user_id": ALICE, "display_name": display_name, "avatar_url": None},
)
+
+ def test_icu_word_boundary_punctuation(self) -> None:
+ """
+ Tests the behaviour of punctuation with the ICU tokeniser.
+
+ Seems to depend on underlying version of ICU.
+ """
+
+ # Note: either tokenisation is fine, because Postgres actually splits
+ # words itself afterwards.
+ self.assertIn(
+ _parse_words_with_icu("lazy'fox jumped:over the.dog"),
+ (
+ # ICU 66 on Ubuntu 20.04
+ ["lazy'fox", "jumped", "over", "the", "dog"],
+ # ICU 70 on Ubuntu 22.04
+ ["lazy'fox", "jumped:over", "the.dog"],
+ ),
+ )
+
+ def test_regex_word_boundary_punctuation(self) -> None:
+ """
+ Tests the behaviour of punctuation with the non-ICU tokeniser
+ """
+ self.assertEqual(
+ _parse_words_with_regex("lazy'fox jumped:over the.dog"),
+ ["lazy", "fox", "jumped", "over", "the", "dog"],
+ )
diff --git a/tests/test_federation.py b/tests/test_federation.py
index 82dfd88b99..46d2f99eac 100644
--- a/tests/test_federation.py
+++ b/tests/test_federation.py
@@ -47,7 +47,7 @@ class MessageAcceptTests(unittest.HomeserverTestCase):
room_creator.create_room(
our_user, room_creator._presets_dict["public_chat"], ratelimit=False
)
- )[0]["room_id"]
+ )[0]
self.store = self.hs.get_datastores().main
diff --git a/tests/test_mau.py b/tests/test_mau.py
index 4e7665a22b..ff21098a59 100644
--- a/tests/test_mau.py
+++ b/tests/test_mau.py
@@ -32,7 +32,6 @@ from tests.utils import default_config
class TestMauLimit(unittest.HomeserverTestCase):
-
servlets = [register.register_servlets, sync.register_servlets]
def default_config(self) -> JsonDict:
|
